Commit e974bef8 authored by Alexey Ternavskiy's avatar Alexey Ternavskiy

Initial commit.

parents
### OSX ###
.DS_Store
.AppleDouble
.LSOverride
### SublimeText ###
# cache files for sublime text
*.tmlanguage.cache
*.tmPreferences.cache
*.stTheme.cache
# workspace files are user-specific
*.sublime-workspace
# project files should be checked into the repository, unless a significant
# proportion of contributors will probably not be using SublimeText
# *.sublime-project
# sftp configuration file
sftp-config.json
# Basics
*.py[cod]
*.pyc
__pycache__
# Logs
*.log
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
htmlcov
# Translations
*.mo
*.pot
# Pycharm
.idea
# Vim
*~
*.swp
*.swo
# npm
node_modules/
# Compass
.sass-cache
# virtual environments
.env
# User-uploaded media
py_digest/media/
#vagrant
.vagrant
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.define "develop" do |develop|
develop.vm.box = "develop"
develop.vm.box_url = "http://vagrant.steel.kiwi/vagrant/develop/"
####Network
develop.vm.network :forwarded_port, guest: 8000, host: 8000
develop.vm.network :private_network, ip: '192.168.80.50'
develop.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ['nolock,vers=3,udp,noatime,actimeo=1']
#develop.vm.synced_folder ".", "/vagrant", type: :virtualbox
####ansible_provision
develop.vm.provision "ansible_local" do |ansible|
ansible.verbose = "v"
ansible.playbook = "ansible/vagrant.yml"
end
####system resources
develop.vm.hostname = "develop"
develop.vm.provider "virtualbox" do |v|
v.memory = 1024
v.cpus = 1
end
end
end
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Inspired from: https://github.com/redhat-openstack/khaleesi/blob/master/plugins/callbacks/human_log.py
# Further improved support Ansible 2.0
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
try:
import simplejson as json
except ImportError:
import json
# Fields to reformat output for
FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout',
'stderr', 'results']
class CallbackModule(object):
"""
Ansible callback plugin for human-readable result logging
"""
CALLBACK_VERSION = 2.0
CALLBACK_TYPE = 'notification'
CALLBACK_NAME = 'human_log'
CALLBACK_NEEDS_WHITELIST = False
def human_log(self, data):
if type(data) == dict:
for field in FIELDS:
no_log = data.get('_ansible_no_log')
if field in data.keys() and data[field] and no_log is not True:
output = self._format_output(data[field])
print("\n")
print("{}: {}".format(field, output.replace("\\n", "\n")))
def _format_output(self, output):
# Strip unicode
if type(output) == unicode:
output = output.encode('ascii', 'replace')
# If output is a dict
if type(output) == dict:
return json.dumps(output, indent=2)
# If output is a list of dicts
if type(output) == list and type(output[0]) == dict:
# This gets a little complicated because it potentially means
# nested results, usually because of with_items.
real_output = list()
for index, item in enumerate(output):
copy = item
if type(item) == dict:
for field in FIELDS:
if field in item.keys():
copy[field] = self._format_output(item[field])
real_output.append(copy)
return json.dumps(output, indent=2)
# If output is a list of strings
if type(output) == list and type(output[0]) != dict:
# Strip newline characters
real_output = list()
for item in output:
if "\n" in item:
for string in item.split("\n"):
real_output.append(string)
else:
real_output.append(item)
# Reformat lists with line breaks only if the total length is
# >75 chars
if len("".join(real_output)) > 75:
return "\n" + "\n".join(real_output)
else:
return " ".join(real_output)
# Otherwise it's a string, (or an int, float, etc.) just return it
return str(output)
def on_any(self, *args, **kwargs):
pass
def runner_on_failed(self, host, res, ignore_errors=False):
self.human_log(res)
def runner_on_ok(self, host, res):
pass
# self.human_log(res)
def runner_on_skipped(self, host, item=None):
pass
def runner_on_unreachable(self, host, res):
self.human_log(res)
def runner_on_no_hosts(self):
pass
def runner_on_async_poll(self, host, res, jid, clock):
self.human_log(res)
def runner_on_async_ok(self, host, res, jid):
self.human_log(res)
def runner_on_async_failed(self, host, res, jid):
self.human_log(res)
def playbook_on_start(self):
pass
def playbook_on_notify(self, host, handler):
pass
def playbook_on_no_hosts_matched(self):
pass
def playbook_on_no_hosts_remaining(self):
pass
def playbook_on_task_start(self, name, is_conditional):
pass
def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
pass
def playbook_on_setup(self):
pass
def playbook_on_import_for_host(self, host, imported_file):
pass
def playbook_on_not_import_for_host(self, host, missing_file):
pass
def playbook_on_play_start(self, name):
pass
def playbook_on_stats(self, stats):
pass
def on_file_diff(self, host, diff):
pass
# V2 METHODS
def v2_on_any(self, *args, **kwargs):
pass
def v2_runner_on_failed(self, result, ignore_errors=False):
self.human_log(result._result)
def v2_runner_on_ok(self, result):
# self.human_log(result._result)
pass
def v2_runner_on_skipped(self, result):
pass
def v2_runner_on_unreachable(self, result):
self.human_log(result._result)
def v2_runner_on_no_hosts(self, task):
pass
def v2_runner_on_async_poll(self, result):
self.human_log(result._result)
def v2_runner_on_async_ok(self, host, result):
self.human_log(result._result)
def v2_runner_on_async_failed(self, result):
self.human_log(result._result)
def v2_playbook_on_start(self, playbook):
pass
def v2_playbook_on_notify(self, result, handler):
pass
def v2_playbook_on_no_hosts_matched(self):
pass
def v2_playbook_on_no_hosts_remaining(self):
pass
def v2_playbook_on_task_start(self, task, is_conditional):
pass
def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None,
encrypt=None, confirm=False, salt_size=None,
salt=None, default=None):
pass
def v2_playbook_on_setup(self):
pass
def v2_playbook_on_import_for_host(self, result, imported_file):
pass
def v2_playbook_on_not_import_for_host(self, result, missing_file):
pass
def v2_playbook_on_play_start(self, play):
pass
def v2_playbook_on_stats(self, stats):
pass
def v2_on_file_diff(self, result):
pass
def v2_playbook_on_item_ok(self, result):
pass
def v2_playbook_on_item_failed(self, result):
pass
def v2_playbook_on_item_skipped(self, result):
pass
def v2_playbook_on_include(self, included_file):
pass
def v2_playbook_item_on_ok(self, result):
pass
def v2_playbook_item_on_failed(self, result):
pass
def v2_playbook_item_on_skipped(self, result):
pass
---
- name: enable apache
service: >
name=apache2
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: apache2
\ No newline at end of file
---
- include: setup_virtualenv.yml
tags: virtualenv
- include: setup_django_app.yml
vars:
run_django_db_migrations: yes
run_django_filldb: yes
tags: deploy
\ No newline at end of file
---
- name: Install packages required by the Django app inside virtualenv
pip: >
requirements={{ requirements_file }}
executable="{{ virtualenv_path }}/bin/pip"
become: no
tags: requirements
- name: Run Django database migrations
django_manage: >
command=migrate
app_path="{{ project_path }}"
virtualenv="{{ virtualenv_path }}"
settings="{{ django_settings }}"
when: run_django_db_migrations is defined and run_django_db_migrations
become: no
tags: django.migrate
---
- name: Create python virtual environment
shell: "virtualenv -p /usr/local/bin/python{{ python_version }} {{ virtualenv_path }}"
become: no
tags: create_venv
- name: Upgrade pip and setuptools
pip: >
name={{ item }}
executable="{{ virtualenv_path }}/bin/pip"
state=latest
with_items:
- pip
- setuptools
become: no
tags: create_venv
---
- name: enable mysql
service: >
name=mysql
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: mysql
- name: create user
mysql_user: >
name={{ db_user }}
login_user=root
login_password=''
password=''
priv=*.*:ALL
state=present
become: no
tags: db
- name: create database
mysql_db: >
name={{ db_name }}
login_user={{ db_user }}
encoding='utf8'
state=present
become: no
tags: db
---
- name: enable nginx
service: >
name=nginx
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: nginx
\ No newline at end of file
---
- name: enable postgresql
service: >
name=postgresql
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: postgresql
- name: Create database
postgresql_db: >
login_user={{ db_user }}
name={{ db_name }}
owner={{ db_user }}
encoding='UTF-8'
lc_collate='en_US.UTF-8'
lc_ctype='en_US.UTF-8'
state=present
become: no
tags: create_db
---
- name: enable rabbitmq
service: >
name=rabbitmq-server
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: rabbitmq
---
- name: enable redis-server
service: >
name=redis-server
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: redis
\ No newline at end of file
---
- name: enable supervisor
service: >
name=supervisor
enabled=yes
runlevel=default
state=started
become: yes
become_user: root
tags: supervisor
\ No newline at end of file
---
- name: Configure box
hosts: all
vars:
- update_apt_cache: yes
vars_files:
- vars/vagrant.yml
roles:
- { role: mysql, when: "mysql == true" }
- { role: apache, when: "apache == true" }
- { role: nginx, when: "nginx == true" }
- { role: postgresql, when: "postgresql == true" }
- { role: rabbitmq, when: "rabbitmq == true" }
- { role: redis, when: "redis == true" }
- { role: supervisor, when: "supervisor == true" }
- { role: app, when: "app == true" }
---
project_name: py_digest
application_name: py_digest
# Database settings.
db_user: "vagrant"
db_name: "{{ application_name }}"
db_password: ""
# Application settings.
virtualenv_path: "/home/vagrant/venv"
project_path: "/vagrant"
requirements_file: "{{ project_path }}/requirements/local.txt"
django_settings_path: "{{ project_path }}/config"
django_settings: "config.settings"
#Python version - change if you needed (2 or 3.5 or 3.6)
python_version: 3.6
# Services
mysql: no
apache: no
nginx: no
postgresql: no
rabbitmq: no
redis: no
supervisor: no
app: true
{"last_post_id": 889433495747080192}
\ No newline at end of file
TWITTER_CONSUMER_KEY=None
TWITTER_CONSUMER_SECRET=None
TWITTER_ACCESS_TOKEN=None
TWITTER_ACCESS_TOKEN_SECRET=None
\ No newline at end of file
import logging
from asyncio import Semaphore, ensure_future, gather, get_event_loop
from json import load, dumps
from urllib.parse import urlparse
import twitter
from aiohttp import ClientSession
from bs4 import BeautifulSoup
from os.path import join as path_join, getsize
import settings
logger = logging.getLogger(__name__)
FILE_PATH = path_join(settings.BASE_DIR, 'data.json')
api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
consumer_secret=settings.TWITTER_CONSUMER_SECRET,
access_token_key=settings.TWITTER_ACCESS_TOKEN,
access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET)
def get_last_post_id():
with open(FILE_PATH) as file:
if not getsize(FILE_PATH):
return None
try:
data = load(file)
return data.get('last_post_id', None)
except Exception as e:
logger.exception(e)
return None
def update_last_post_id(last_post_id):
with open(FILE_PATH, 'w+') as file:
try:
data = dumps({'last_post_id': last_post_id})
file.write(data)
except Exception as e:
logger.exception(e)
return False
return True
def get_user_id():
user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
return user.id
async def fetch(obj, session):
text = obj.text.split(':')[0]
url = obj.urls[0]._json.get('expanded_url')
url_parse_data = urlparse(url)
if url_parse_data.hostname == 'pythondigest.ru' and 'issue' in url_parse_data.path:
return f'{text}: {url}'
async with session.get(url) as response:
soup = BeautifulSoup(await response.text(), 'lxml')
elem = soup.find('a', 'btn btn-primary btn-sm pull-left')
if hasattr(elem, 'attrs'):
link = elem.attrs.get('href')