Commit 0280dbf2 authored by Alexey Ternavskiy's avatar Alexey Ternavskiy

Minor fixes.

parent d95be5fc
...@@ -62,3 +62,5 @@ py_digest/media/ ...@@ -62,3 +62,5 @@ py_digest/media/
#vagrant #vagrant
.vagrant .vagrant
### Py_digest parser
#### Script fetch the data from twitter and send to specific slack channel
1. clone project `git clone`
2. go into folder `cd py_digest`
3. create virtual environment
4. activate virtual environment
5. install requirements `pip install -r requirements.txt`
6. create new .env file `cp env.example .env`
7. filling .env file
6. run script `python`
May be better way its run this script each hour through cron.
{"last_post_id": 892295179343994880}
\ No newline at end of file
from py_digest.slack import write_into_channel from py_digest.slack import write_into_channel
from py_digest.twitter import get_user_posts from py_digest.twitter import get_posts
if __name__ == '__main__': if __name__ == '__main__':
posts = get_user_posts() posts = get_posts()
if posts: if posts:
for post in reversed(posts): for post in reversed(posts):
write_into_channel(post) write_into_channel(post)
...@@ -2,10 +2,12 @@ from slackclient import SlackClient ...@@ -2,10 +2,12 @@ from slackclient import SlackClient
import settings import settings
slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN)
slack_client_bot = SlackClient(settings.SLACK_BOT_TOKEN)
def check_channel(): def check_channel():
slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN) response = slack_client.api_call('channels.list', exclude_archived=True)
response = slack_client.api_call('channels.list', exclude_archived=1)
if 'ok' in response: if 'ok' in response:
channels = response.get('channels') channels = response.get('channels')
slack_channel = [ slack_channel = [
...@@ -25,8 +27,4 @@ channel = check_channel() ...@@ -25,8 +27,4 @@ channel = check_channel()
def write_into_channel(post): def write_into_channel(post):
slack_client = SlackClient(settings.SLACK_BOT_TOKEN) slack_client_bot.api_call('chat.postMessage', channel=channel.get('id'), text=post)
from .api import get_user_posts from .api import get_posts
\ No newline at end of file \ No newline at end of file
...@@ -7,7 +7,7 @@ import twitter ...@@ -7,7 +7,7 @@ import twitter
import uvloop import uvloop
from aiohttp import ClientSession from aiohttp import ClientSession
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from os.path import join as path_join, getsize from os.path import join as path_join
import settings import settings
...@@ -15,7 +15,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) ...@@ -15,7 +15,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
FILE_PATH = path_join(settings.BASE_DIR, 'data.json') FILE_PATH = path_join(settings.BASE_DIR, 'db.json')
api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY, api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
consumer_secret=settings.TWITTER_CONSUMER_SECRET, consumer_secret=settings.TWITTER_CONSUMER_SECRET,
...@@ -23,32 +23,40 @@ api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY, ...@@ -23,32 +23,40 @@ api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET) access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET)
def _get_from_file(key):
with open(FILE_PATH, 'r') as file:
data = load(file)
return data.get(key, None)
except Exception as e:
return None
except FileNotFoundError:
return None
def get_last_post_id(): def get_last_post_id():
with open(FILE_PATH, 'w') as file: return _get_from_file('last_post_id')
if not getsize(FILE_PATH):
return None
try: def get_user_id():
data = load(file) user_id = _get_from_file('user_id')
return data.get('last_post_id', None) if user_id:
except Exception as e: return user_id
logger.exception(e) user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
return None return
def update_last_post_id(last_post_id): def update_last_post_id(last_post_id, user_id):
with open(FILE_PATH, 'w+') as file: with open(FILE_PATH, 'w+') as file:
try: try:
data = dumps({'last_post_id': last_post_id}) data = dumps({'last_post_id': last_post_id, 'user_id': user_id})
file.write(data) file.write(data)
return True
except Exception as e: except Exception as e:
logger.exception(e) logger.exception(e)
return False return False
return True
def get_user_id():
user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
async def fetch(obj, session): async def fetch(obj, session):
...@@ -81,15 +89,19 @@ async def run(urls): ...@@ -81,15 +89,19 @@ async def run(urls):
return await responses return await responses
def get_user_posts(): def get_posts():
last_post_id = get_last_post_id() last_post_id = get_last_post_id()
user_id = get_user_id() user_id = get_user_id()
results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id) results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id, count=50)
if not results: if not results:
return None return None
update_last_post_id(results[0].id) update_last_post_id(results[0].id, user_id)
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(results)) future = asyncio.ensure_future(run(results))
posts = loop.run_until_complete(future) try:
loop.close() posts = loop.run_until_complete(future)
return posts return posts
except Exception as e:
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment