Initial commit.

.gitignore

+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+
+### SublimeText ###
+# cache files for sublime text
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+
+# workspace files are user-specific
+*.sublime-workspace
+
+# project files should be checked into the repository, unless a significant
+# proportion of contributors will probably not be using SublimeText
+# *.sublime-project
+
+# sftp configuration file
+sftp-config.json
+
+# Basics
+*.py[cod]
+*.pyc
+__pycache__
+
+# Logs
+*.log
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+htmlcov
+
+# Translations
+*.mo
+*.pot
+
+# Pycharm
+.idea
+
+# Vim
+
+*~
+*.swp
+*.swo
+
+# npm
+node_modules/
+
+# Compass
+.sass-cache
+
+# virtual environments
+.env
+
+# User-uploaded media
+py_digest/media/
+
+
+#vagrant
+.vagrant

README.md

+

Vagrantfile

+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+	config.vm.define "develop" do |develop|
+		develop.vm.box = "develop"
+		develop.vm.box_url = "http://vagrant.steel.kiwi/vagrant/develop/"
+
+	####Network
+		develop.vm.network :forwarded_port, guest: 8000, host: 8000
+		develop.vm.network :private_network, ip: '192.168.80.50'
+		develop.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ['nolock,vers=3,udp,noatime,actimeo=1']
+		#develop.vm.synced_folder ".", "/vagrant", type: :virtualbox
+
+	####ansible_provision
+		develop.vm.provision "ansible_local" do |ansible|
+		ansible.verbose = "v"
+		ansible.playbook = "ansible/vagrant.yml"
+		end
+
+	####system resources
+		develop.vm.hostname = "develop"
+		  develop.vm.provider "virtualbox" do |v|
+		  v.memory = 1024
+		  v.cpus = 1
+		  end
+	end
+end

ansible/callback_plugins/human_log.py

+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Inspired from: https://github.com/redhat-openstack/khaleesi/blob/master/plugins/callbacks/human_log.py
+# Further improved support Ansible 2.0
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+# Fields to reformat output for
+FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout',
+          'stderr', 'results']
+
+
+class CallbackModule(object):
+
+    """
+    Ansible callback plugin for human-readable result logging
+    """
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'notification'
+    CALLBACK_NAME = 'human_log'
+    CALLBACK_NEEDS_WHITELIST = False
+
+    def human_log(self, data):
+        if type(data) == dict:
+            for field in FIELDS:
+                no_log = data.get('_ansible_no_log')
+                if field in data.keys() and data[field] and no_log is not True:
+                    output = self._format_output(data[field])
+                    print("\n")
+                    print("{}: {}".format(field, output.replace("\\n", "\n")))
+
+    def _format_output(self, output):
+        # Strip unicode
+        if type(output) == unicode:
+            output = output.encode('ascii', 'replace')
+
+        # If output is a dict
+        if type(output) == dict:
+            return json.dumps(output, indent=2)
+
+        # If output is a list of dicts
+        if type(output) == list and type(output[0]) == dict:
+            # This gets a little complicated because it potentially means
+            # nested results, usually because of with_items.
+            real_output = list()
+            for index, item in enumerate(output):
+                copy = item
+                if type(item) == dict:
+                    for field in FIELDS:
+                        if field in item.keys():
+                            copy[field] = self._format_output(item[field])
+                real_output.append(copy)
+            return json.dumps(output, indent=2)
+
+        # If output is a list of strings
+        if type(output) == list and type(output[0]) != dict:
+            # Strip newline characters
+            real_output = list()
+            for item in output:
+                if "\n" in item:
+                    for string in item.split("\n"):
+                        real_output.append(string)
+                else:
+                    real_output.append(item)
+
+            # Reformat lists with line breaks only if the total length is
+            # >75 chars
+            if len("".join(real_output)) > 75:
+                return "\n" + "\n".join(real_output)
+            else:
+                return " ".join(real_output)
+
+        # Otherwise it's a string, (or an int, float, etc.) just return it
+        return str(output)
+
+    def on_any(self, *args, **kwargs):
+        pass
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        self.human_log(res)
+
+    def runner_on_ok(self, host, res):
+        pass
+        # self.human_log(res)
+
+    def runner_on_skipped(self, host, item=None):
+        pass
+
+    def runner_on_unreachable(self, host, res):
+        self.human_log(res)
+
+    def runner_on_no_hosts(self):
+        pass
+
+    def runner_on_async_poll(self, host, res, jid, clock):
+        self.human_log(res)
+
+    def runner_on_async_ok(self, host, res, jid):
+        self.human_log(res)
+
+    def runner_on_async_failed(self, host, res, jid):
+        self.human_log(res)
+
+    def playbook_on_start(self):
+        pass
+
+    def playbook_on_notify(self, host, handler):
+        pass
+
+    def playbook_on_no_hosts_matched(self):
+        pass
+
+    def playbook_on_no_hosts_remaining(self):
+        pass
+
+    def playbook_on_task_start(self, name, is_conditional):
+        pass
+
+    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
+        pass
+
+    def playbook_on_setup(self):
+        pass
+
+    def playbook_on_import_for_host(self, host, imported_file):
+        pass
+
+    def playbook_on_not_import_for_host(self, host, missing_file):
+        pass
+
+    def playbook_on_play_start(self, name):
+        pass
+
+    def playbook_on_stats(self, stats):
+        pass
+
+    def on_file_diff(self, host, diff):
+        pass
+
+
+# V2 METHODS
+    def v2_on_any(self, *args, **kwargs):
+        pass
+
+    def v2_runner_on_failed(self, result, ignore_errors=False):
+        self.human_log(result._result)
+
+    def v2_runner_on_ok(self, result):
+        # self.human_log(result._result)
+        pass
+
+    def v2_runner_on_skipped(self, result):
+        pass
+
+    def v2_runner_on_unreachable(self, result):
+        self.human_log(result._result)
+
+    def v2_runner_on_no_hosts(self, task):
+        pass
+
+    def v2_runner_on_async_poll(self, result):
+        self.human_log(result._result)
+
+    def v2_runner_on_async_ok(self, host, result):
+        self.human_log(result._result)
+
+    def v2_runner_on_async_failed(self, result):
+        self.human_log(result._result)
+
+    def v2_playbook_on_start(self, playbook):
+        pass
+
+    def v2_playbook_on_notify(self, result, handler):
+        pass
+
+    def v2_playbook_on_no_hosts_matched(self):
+        pass
+
+    def v2_playbook_on_no_hosts_remaining(self):
+        pass
+
+    def v2_playbook_on_task_start(self, task, is_conditional):
+        pass
+
+    def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None,
+                                   encrypt=None, confirm=False, salt_size=None,
+                                   salt=None, default=None):
+        pass
+
+    def v2_playbook_on_setup(self):
+        pass
+
+    def v2_playbook_on_import_for_host(self, result, imported_file):
+        pass
+
+    def v2_playbook_on_not_import_for_host(self, result, missing_file):
+        pass
+
+    def v2_playbook_on_play_start(self, play):
+        pass
+
+    def v2_playbook_on_stats(self, stats):
+        pass
+
+    def v2_on_file_diff(self, result):
+        pass
+
+    def v2_playbook_on_item_ok(self, result):
+        pass
+
+    def v2_playbook_on_item_failed(self, result):
+        pass
+
+    def v2_playbook_on_item_skipped(self, result):
+        pass
+
+    def v2_playbook_on_include(self, included_file):
+        pass
+
+    def v2_playbook_item_on_ok(self, result):
+        pass
+
+    def v2_playbook_item_on_failed(self, result):
+        pass
+
+    def v2_playbook_item_on_skipped(self, result):
+        pass

ansible/roles/apache/tasks/main.yml

+---
+
+- name: enable apache
+  service: >
+    name=apache2
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: apache2
\ No newline at end of file

ansible/roles/app/tasks/main.yml

+---
+
+- include: setup_virtualenv.yml
+  tags: virtualenv
+
+- include: setup_django_app.yml
+  vars:
+    run_django_db_migrations: yes
+    run_django_filldb: yes
+  tags: deploy
\ No newline at end of file

ansible/roles/app/tasks/setup_django_app.yml

+---
+
+- name: Install packages required by the Django app inside virtualenv
+  pip: >
+    requirements={{ requirements_file }}
+    executable="{{ virtualenv_path }}/bin/pip"
+  become: no
+  tags: requirements
+
+- name: Run Django database migrations
+  django_manage: >
+    command=migrate
+    app_path="{{ project_path }}"
+    virtualenv="{{ virtualenv_path }}"
+    settings="{{ django_settings }}"
+  when: run_django_db_migrations is defined and run_django_db_migrations
+  become: no
+  tags: django.migrate
+

ansible/roles/app/tasks/setup_virtualenv.yml

+---
+
+- name: Create python virtual environment
+  shell: "virtualenv -p /usr/local/bin/python{{ python_version }} {{ virtualenv_path }}"
+  become: no
+  tags: create_venv
+
+- name: Upgrade pip and setuptools
+  pip: >
+    name={{ item }}
+    executable="{{ virtualenv_path }}/bin/pip"
+    state=latest
+  with_items:
+    - pip
+    - setuptools
+  become: no
+  tags: create_venv
+

ansible/roles/mysql/tasks/main.yml

+---
+
+- name: enable mysql
+  service: >
+    name=mysql
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: mysql
+
+- name: create user
+  mysql_user: >
+    name={{ db_user }}
+    login_user=root
+    login_password=''
+    password=''
+    priv=*.*:ALL
+    state=present
+  become: no
+  tags: db
+
+- name: create database
+  mysql_db: >
+    name={{ db_name }}
+    login_user={{ db_user }}
+    encoding='utf8'
+    state=present
+  become: no
+  tags: db
+

ansible/roles/nginx/tasks/main.yml

+---
+
+- name: enable nginx
+  service: >
+    name=nginx
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: nginx
\ No newline at end of file

ansible/roles/postgresql/tasks/main.yml

+---
+
+- name: enable postgresql
+  service: >
+    name=postgresql
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: postgresql
+
+- name: Create database
+  postgresql_db: >
+    login_user={{ db_user }}
+    name={{ db_name }}
+    owner={{ db_user }}
+    encoding='UTF-8'
+    lc_collate='en_US.UTF-8'
+    lc_ctype='en_US.UTF-8'
+    state=present
+  become: no
+  tags: create_db

ansible/roles/rabbitmq/tasks/main.yml

+---
+
+- name: enable rabbitmq
+  service: >
+    name=rabbitmq-server
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: rabbitmq

ansible/roles/redis/tasks/main.yml

+---
+
+- name: enable redis-server
+  service: >
+    name=redis-server
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: redis
\ No newline at end of file

ansible/roles/supervisor/tasks/main.yml

+---
+
+- name: enable supervisor
+  service: >
+    name=supervisor
+    enabled=yes
+    runlevel=default
+    state=started
+  become: yes
+  become_user: root
+  tags: supervisor
\ No newline at end of file

ansible/vagrant.retry

+develop

ansible/vagrant.yml

+---
+- name: Configure box
+
+  hosts: all
+  vars:
+    - update_apt_cache: yes
+  vars_files:
+    - vars/vagrant.yml
+
+  roles:
+    - { role: mysql, when: "mysql == true" }
+    - { role: apache, when: "apache == true" }
+    - { role: nginx, when: "nginx == true" }
+    - { role: postgresql, when: "postgresql == true" }
+    - { role: rabbitmq, when: "rabbitmq == true" }
+    - { role: redis, when: "redis == true" }
+    - { role: supervisor, when: "supervisor == true" }
+    - { role: app, when: "app == true" }
+

ansible/vars/vagrant.yml

+---
+project_name: py_digest
+application_name: py_digest
+
+
+# Database settings.
+db_user: "vagrant"
+db_name: "{{ application_name }}"
+db_password: ""
+
+# Application settings.
+virtualenv_path: "/home/vagrant/venv"
+project_path: "/vagrant"
+requirements_file: "{{ project_path }}/requirements/local.txt"
+django_settings_path: "{{ project_path }}/config"
+django_settings: "config.settings"
+
+
+#Python version - change if you needed (2 or 3.5 or 3.6)
+python_version: 3.6
+
+# Services
+mysql: no
+apache: no
+nginx: no
+postgresql: no
+rabbitmq: no
+redis: no
+supervisor: no
+app: true

data.json

+{"last_post_id": 889433495747080192}
\ No newline at end of file

env.example

+TWITTER_CONSUMER_KEY=None
+TWITTER_CONSUMER_SECRET=None
+TWITTER_ACCESS_TOKEN=None
+TWITTER_ACCESS_TOKEN_SECRET=None
\ No newline at end of file

py_digest/twitter/api.py

+import logging
+from asyncio import Semaphore, ensure_future, gather, get_event_loop
+from json import load, dumps
+from urllib.parse import urlparse
+
+import twitter
+from aiohttp import ClientSession
+from bs4 import BeautifulSoup
+from os.path import join as path_join, getsize
+
+import settings
+
+logger = logging.getLogger(__name__)
+
+FILE_PATH = path_join(settings.BASE_DIR, 'data.json')
+
+api = twitter.Api(consumer_key=settings.TWITTER_CONSUMER_KEY,
+                  consumer_secret=settings.TWITTER_CONSUMER_SECRET,
+                  access_token_key=settings.TWITTER_ACCESS_TOKEN,
+                  access_token_secret=settings.TWITTER_ACCESS_TOKEN_SECRET)
+
+
+def get_last_post_id():
+    with open(FILE_PATH) as file:
+        if not getsize(FILE_PATH):
+            return None
+        try:
+            data = load(file)
+            return data.get('last_post_id', None)
+        except Exception as e:
+            logger.exception(e)
+            return None
+
+
+def update_last_post_id(last_post_id):
+    with open(FILE_PATH, 'w+') as file:
+        try:
+            data = dumps({'last_post_id': last_post_id})
+            file.write(data)
+        except Exception as e:
+            logger.exception(e)
+            return False
+    return True
+
+
+def get_user_id():
+    user = api.GetUser(screen_name=settings.PYDIGEST_USER_NAME)
+    return user.id
+
+
+async def fetch(obj, session):
+    text = obj.text.split(':')[0]
+    url = obj.urls[0]._json.get('expanded_url')
+    url_parse_data = urlparse(url)
+    if url_parse_data.hostname == 'pythondigest.ru' and 'issue' in url_parse_data.path:
+        return f'{text}: {url}'
+    async with session.get(url) as response:
+        soup = BeautifulSoup(await response.text(), 'lxml')
+        elem = soup.find('a', 'btn btn-primary btn-sm pull-left')
+        if hasattr(elem, 'attrs'):
+            link = elem.attrs.get('href')