Minor fixes.

.gitignore

@@ -62,3 +62,5 @@ py_digest/media/
 
 #vagrant
 .vagrant
+
+db.json

README.md

+### Py_digest parser
+#### Script fetch the data from twitter and send to specific slack channel
+#####Install
 
+1. clone project `git clone https://git.steelkiwi.com/ternavskiy/py_digest.git`
+2. go into folder `cd py_digest`
+3. create virtual environment
+4. activate virtual environment
+5. install requirements `pip install -r requirements.txt`
+6. create new .env file `cp env.example .env`
+7. filling .env file
+6. run script `python main.py`
+
+May be better way its run this script each hour through cron.

data.json

-{"last_post_id": 892295179343994880}
\ No newline at end of file

main.py

 from py_digest.slack import write_into_channel
-from py_digest.twitter import get_user_posts
+from py_digest.twitter import get_posts
 
 if __name__ == '__main__':
-    posts = get_user_posts()
+    posts = get_posts()
     if posts:
         for post in reversed(posts):
             write_into_channel(post)

py_digest/slack/api.py

@@ -2,10 +2,12 @@ from slackclient import SlackClient
 
 import settings
 
+slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN)
+slack_client_bot = SlackClient(settings.SLACK_BOT_TOKEN)
+
 
 def check_channel():
-    slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN)
-    response = slack_client.api_call('channels.list', exclude_archived=1)
+    response = slack_client.api_call('channels.list', exclude_archived=True)
     if 'ok' in response:
         channels = response.get('channels')
         slack_channel = [
@@ -25,8 +27,4 @@ channel = check_channel()
 
 
 def write_into_channel(post):
-    slack_client = SlackClient(settings.SLACK_BOT_TOKEN)
-    slack_client.api_call('chat.postMessage',
-                          channel=channel.get('id'),
-                          text=post,
-                          as_user=False)
+    slack_client_bot.api_call('chat.postMessage', channel=channel.get('id'), text=post)

py_digest/twitter/__init__.py

-from .api import get_user_posts
\ No newline at end of file
+from .api import get_posts
\ No newline at end of file

py_digest/twitter/api.py

@@ -7,7 +7,7 @@ import twitter
 import uvloop
 from aiohttp import ClientSession
 from bs4 import BeautifulSoup
-from os.path import join as path_join, getsize
+from os.path import join as path_join
 
 import settings
 
@@ -15,7 +15,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 logger = logging.getLogger(__name__)
 
-FILE_PATH = path_join(settings.BASE_DIR, 'data.json')
+FILE_PATH = path_join(settings.BASE_DIR, 'db.json')
 
 api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
                   consumer_secret=settings.TWITTER_CONSUMER_SECRET,
@@ -23,32 +23,40 @@ api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
                   access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET)
 
 
+def _get_from_file(key):
+    try:
+        with open(FILE_PATH, 'r') as file:
+            try:
+                data = load(file)
+                return data.get(key, None)
+            except Exception as e:
+                logger.exception(e)
+                return None
+    except FileNotFoundError:
+        return None
+
+
 def get_last_post_id():
-    with open(FILE_PATH, 'w') as file:
-        if not getsize(FILE_PATH):
-            return None
-        try:
-            data = load(file)
-            return data.get('last_post_id', None)
-        except Exception as e:
-            logger.exception(e)
-            return None
+    return _get_from_file('last_post_id')
+
+
+def get_user_id():
+    user_id = _get_from_file('user_id')
+    if user_id:
+        return user_id
+    user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
+    return user.id
 
 
-def update_last_post_id(last_post_id):
+def update_last_post_id(last_post_id, user_id):
     with open(FILE_PATH, 'w+') as file:
         try:
-            data = dumps({'last_post_id': last_post_id})
+            data = dumps({'last_post_id': last_post_id, 'user_id': user_id})
             file.write(data)
+            return True
         except Exception as e:
             logger.exception(e)
             return False
-    return True
-
-
-def get_user_id():
-    user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
-    return user.id
 
 
 async def fetch(obj, session):
@@ -81,15 +89,19 @@ async def run(urls):
         return await responses
 
 
-def get_user_posts():
+def get_posts():
     last_post_id = get_last_post_id()
     user_id = get_user_id()
-    results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id)
+    results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id, count=50)
     if not results:
         return None
-    update_last_post_id(results[0].id)
+    update_last_post_id(results[0].id, user_id)
     loop = asyncio.get_event_loop()
     future = asyncio.ensure_future(run(results))
-    posts = loop.run_until_complete(future)
-    loop.close()
-    return posts
+    try:
+        posts = loop.run_until_complete(future)
+        return posts
+    except Exception as e:
+        logger.exception(e)
+    finally:
+        loop.close()