From 0280dbf2e2c3065b00dce75121c833e367606135 Mon Sep 17 00:00:00 2001 From: Alexey Ternavskiy Date: Thu, 3 Aug 2017 15:12:35 +0300 Subject: [PATCH] Minor fixes. --- .gitignore | 2 ++ README.md | 13 ++++++++ data.json | 1 - main.py | 4 +-- py_digest/slack/api.py | 12 +++---- py_digest/twitter/__init__.py | 2 +- py_digest/twitter/api.py | 62 +++++++++++++++++++++-------------- 7 files changed, 60 insertions(+), 36 deletions(-) delete mode 100644 data.json diff --git a/.gitignore b/.gitignore index 7481a91..859acf6 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,5 @@ py_digest/media/ #vagrant .vagrant + +db.json diff --git a/README.md b/README.md index 8b13789..ea5c972 100644 --- a/README.md +++ b/README.md @@ -1 +1,14 @@ +### Py_digest parser +#### Script fetch the data from twitter and send to specific slack channel +#####Install +1. clone project `git clone https://git.steelkiwi.com/ternavskiy/py_digest.git` +2. go into folder `cd py_digest` +3. create virtual environment +4. activate virtual environment +5. install requirements `pip install -r requirements.txt` +6. create new .env file `cp env.example .env` +7. filling .env file +6. run script `python main.py` + +May be better way its run this script each hour through cron. diff --git a/data.json b/data.json deleted file mode 100644 index 56e6d5f..0000000 --- a/data.json +++ /dev/null @@ -1 +0,0 @@ -{"last_post_id": 892295179343994880} \ No newline at end of file diff --git a/main.py b/main.py index 332e181..27d0871 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,8 @@ from py_digest.slack import write_into_channel -from py_digest.twitter import get_user_posts +from py_digest.twitter import get_posts if __name__ == '__main__': - posts = get_user_posts() + posts = get_posts() if posts: for post in reversed(posts): write_into_channel(post) diff --git a/py_digest/slack/api.py b/py_digest/slack/api.py index 05596fb..e6c7d61 100644 --- a/py_digest/slack/api.py +++ b/py_digest/slack/api.py @@ -2,10 +2,12 @@ from slackclient import SlackClient import settings +slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN) +slack_client_bot = SlackClient(settings.SLACK_BOT_TOKEN) + def check_channel(): - slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN) - response = slack_client.api_call('channels.list', exclude_archived=1) + response = slack_client.api_call('channels.list', exclude_archived=True) if 'ok' in response: channels = response.get('channels') slack_channel = [ @@ -25,8 +27,4 @@ channel = check_channel() def write_into_channel(post): - slack_client = SlackClient(settings.SLACK_BOT_TOKEN) - slack_client.api_call('chat.postMessage', - channel=channel.get('id'), - text=post, - as_user=False) + slack_client_bot.api_call('chat.postMessage', channel=channel.get('id'), text=post) diff --git a/py_digest/twitter/__init__.py b/py_digest/twitter/__init__.py index 9c9b4a3..f3f62d0 100644 --- a/py_digest/twitter/__init__.py +++ b/py_digest/twitter/__init__.py @@ -1 +1 @@ -from .api import get_user_posts \ No newline at end of file +from .api import get_posts \ No newline at end of file diff --git a/py_digest/twitter/api.py b/py_digest/twitter/api.py index aea00de..3a3dd41 100644 --- a/py_digest/twitter/api.py +++ b/py_digest/twitter/api.py @@ -7,7 +7,7 @@ import twitter import uvloop from aiohttp import ClientSession from bs4 import BeautifulSoup -from os.path import join as path_join, getsize +from os.path import join as path_join import settings @@ -15,7 +15,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) logger = logging.getLogger(__name__) -FILE_PATH = path_join(settings.BASE_DIR, 'data.json') +FILE_PATH = path_join(settings.BASE_DIR, 'db.json') api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY, consumer_secret=settings.TWITTER_CONSUMER_SECRET, @@ -23,32 +23,40 @@ api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY, access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET) +def _get_from_file(key): + try: + with open(FILE_PATH, 'r') as file: + try: + data = load(file) + return data.get(key, None) + except Exception as e: + logger.exception(e) + return None + except FileNotFoundError: + return None + + def get_last_post_id(): - with open(FILE_PATH, 'w') as file: - if not getsize(FILE_PATH): - return None - try: - data = load(file) - return data.get('last_post_id', None) - except Exception as e: - logger.exception(e) - return None + return _get_from_file('last_post_id') + + +def get_user_id(): + user_id = _get_from_file('user_id') + if user_id: + return user_id + user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME) + return user.id -def update_last_post_id(last_post_id): +def update_last_post_id(last_post_id, user_id): with open(FILE_PATH, 'w+') as file: try: - data = dumps({'last_post_id': last_post_id}) + data = dumps({'last_post_id': last_post_id, 'user_id': user_id}) file.write(data) + return True except Exception as e: logger.exception(e) return False - return True - - -def get_user_id(): - user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME) - return user.id async def fetch(obj, session): @@ -81,15 +89,19 @@ async def run(urls): return await responses -def get_user_posts(): +def get_posts(): last_post_id = get_last_post_id() user_id = get_user_id() - results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id) + results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id, count=50) if not results: return None - update_last_post_id(results[0].id) + update_last_post_id(results[0].id, user_id) loop = asyncio.get_event_loop() future = asyncio.ensure_future(run(results)) - posts = loop.run_until_complete(future) - loop.close() - return posts + try: + posts = loop.run_until_complete(future) + return posts + except Exception as e: + logger.exception(e) + finally: + loop.close() -- GitLab