Commit 0280dbf2 authored by Alexey Ternavskiy's avatar Alexey Ternavskiy

Minor fixes.

parent d95be5fc
......@@ -62,3 +62,5 @@ py_digest/media/
#vagrant
.vagrant
db.json
### Py_digest parser
#### Script fetch the data from twitter and send to specific slack channel
#####Install
1. clone project `git clone https://git.steelkiwi.com/ternavskiy/py_digest.git`
2. go into folder `cd py_digest`
3. create virtual environment
4. activate virtual environment
5. install requirements `pip install -r requirements.txt`
6. create new .env file `cp env.example .env`
7. filling .env file
6. run script `python main.py`
May be better way its run this script each hour through cron.
{"last_post_id": 892295179343994880}
\ No newline at end of file
from py_digest.slack import write_into_channel
from py_digest.twitter import get_user_posts
from py_digest.twitter import get_posts
if __name__ == '__main__':
posts = get_user_posts()
posts = get_posts()
if posts:
for post in reversed(posts):
write_into_channel(post)
......@@ -2,10 +2,12 @@ from slackclient import SlackClient
import settings
slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN)
slack_client_bot = SlackClient(settings.SLACK_BOT_TOKEN)
def check_channel():
slack_client = SlackClient(settings.SLACK_OAUTH_ACCESS_TOKEN)
response = slack_client.api_call('channels.list', exclude_archived=1)
response = slack_client.api_call('channels.list', exclude_archived=True)
if 'ok' in response:
channels = response.get('channels')
slack_channel = [
......@@ -25,8 +27,4 @@ channel = check_channel()
def write_into_channel(post):
slack_client = SlackClient(settings.SLACK_BOT_TOKEN)
slack_client.api_call('chat.postMessage',
channel=channel.get('id'),
text=post,
as_user=False)
slack_client_bot.api_call('chat.postMessage', channel=channel.get('id'), text=post)
from .api import get_user_posts
\ No newline at end of file
from .api import get_posts
\ No newline at end of file
......@@ -7,7 +7,7 @@ import twitter
import uvloop
from aiohttp import ClientSession
from bs4 import BeautifulSoup
from os.path import join as path_join, getsize
from os.path import join as path_join
import settings
......@@ -15,7 +15,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
logger = logging.getLogger(__name__)
FILE_PATH = path_join(settings.BASE_DIR, 'data.json')
FILE_PATH = path_join(settings.BASE_DIR, 'db.json')
api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
consumer_secret=settings.TWITTER_CONSUMER_SECRET,
......@@ -23,32 +23,40 @@ api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET)
def _get_from_file(key):
try:
with open(FILE_PATH, 'r') as file:
try:
data = load(file)
return data.get(key, None)
except Exception as e:
logger.exception(e)
return None
except FileNotFoundError:
return None
def get_last_post_id():
with open(FILE_PATH, 'w') as file:
if not getsize(FILE_PATH):
return None
try:
data = load(file)
return data.get('last_post_id', None)
except Exception as e:
logger.exception(e)
return None
return _get_from_file('last_post_id')
def get_user_id():
user_id = _get_from_file('user_id')
if user_id:
return user_id
user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
return user.id
def update_last_post_id(last_post_id):
def update_last_post_id(last_post_id, user_id):
with open(FILE_PATH, 'w+') as file:
try:
data = dumps({'last_post_id': last_post_id})
data = dumps({'last_post_id': last_post_id, 'user_id': user_id})
file.write(data)
return True
except Exception as e:
logger.exception(e)
return False
return True
def get_user_id():
user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
return user.id
async def fetch(obj, session):
......@@ -81,15 +89,19 @@ async def run(urls):
return await responses
def get_user_posts():
def get_posts():
last_post_id = get_last_post_id()
user_id = get_user_id()
results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id)
results = api.GetUserTimeline(user_id=user_id, since_id=last_post_id, count=50)
if not results:
return None
update_last_post_id(results[0].id)
update_last_post_id(results[0].id, user_id)
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run(results))
posts = loop.run_until_complete(future)
loop.close()
return posts
try:
posts = loop.run_until_complete(future)
return posts
except Exception as e:
logger.exception(e)
finally:
loop.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment