Vivekananda_session4_4VP20CS069_Rahul

Workshop4- Homework/myworld/docker-compose.yml

@@ -0,0 +1,66 @@
+version: "3"
+services:
+ web_service:
+   build:
+     context: ./
+     dockerfile: ./dockerfiles/Dockerfile
+   image: workshop1_web
+   container_name: workshop_web_container
+   stdin_open: true #  docker attach container_id
+   tty: true
+   environment:
+     - RABBITMQ_DEFAULT_USER=myuser
+     - RABBITMQ_DEFAULT_PASS=mypassword
+     - BROKER_HOST=service-rabbitmq
+     - RABBITMQ_DEFAULT_VHOST=extractor
+     - BROKER_PORT=5672
+   ports:
+    - "8000:8000"
+   volumes:
+    - .:/root/workspace/site
+
+ psql-db:
+   image: 'postgres:14'
+   container_name: psql-db
+   environment:
+     - PGPASSWORD=123456
+     - POSTGRES_USER=postgres
+     - POSTGRES_PASSWORD=123456
+   ports:
+     - '5446:5432'
+   volumes:
+     - db:/var/lib/postgresql/data
+
+ service-rabbitmq:
+   container_name: "service_rabbitmq"
+   image: rabbitmq:3.8-management-alpine
+   environment:
+     - RABBITMQ_DEFAULT_USER=myuser
+     - RABBITMQ_DEFAULT_PASS=mypassword
+     - RABBITMQ_DEFAULT_VHOST=extractor
+     - BROKER_HOST=service-rabbitmq
+   ports:
+     - '5673:5673'
+     - '15676:15672'
+ worker:
+   build:
+     context: ./
+     dockerfile: ./dockerfiles/Dockerfile
+   image: workshop1_web
+   container_name: worker
+   stdin_open: true #  docker attach container_id
+   tty: true
+   environment:
+     - RABBITMQ_DEFAULT_USER=myuser
+     - RABBITMQ_DEFAULT_PASS=mypassword
+     - BROKER_HOST=service-rabbitmq
+     - RABBITMQ_DEFAULT_VHOST=extractor
+     - BROKER_PORT=5672
+   ports:
+     - "4356:8000"
+   volumes:
+     - .:/root/workspace/site
+
+volumes:
+ db:
+   driver: local

Workshop4- Homework/myworld/dockerfiles/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.10.2-alpine3.15
+# Install required packages
+# For psycopg2
+RUN apk update && \
+apk --no-cache add --virtual build-deps-alpine build-base && \
+apk --no-cache add --virtual postgresql-deps libpq-dev
+# Install requirements
+RUN pip install --upgrade pip
+RUN pip install Django psycopg2==2.9.3 bs4 html5lib requests python-dateutil celery pytz
+# Create directories
+RUN mkdir -p /root/workspace/src
+COPY ./  /root/workspace/site
+# Switch to project directory
+WORKDIR /root/workspace/site

Workshop4- Homework/myworld/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

Workshop4- Homework/myworld/members/admin.py

@@ -0,0 +1,52 @@
+from django.contrib import admin
+from .models import Students, Blog, Job, JobLogs, JobStats
+from django.urls import reverse
+from django.utils.html import format_html
+
+class DjStudentAdmin(admin.ModelAdmin):
+   list_display = ("first_name", "last_name", "address", "roll_number", "mobile", "branch")
+   list_filter = ("branch",)
+
+class DjBlogAdmin(admin.ModelAdmin):
+  list_display = ("title", "release_date", "blog_time", "author", "created_date")
+  list_filter = ("author",)
+
+
+class DjJob(admin.ModelAdmin):
+  def run(self, obj):
+    return format_html('<a class="button" href="{}">RUN</a>', reverse('scraping', args=(str(obj.pk))))
+
+  def view_stats(self, obj):
+    path = "../jobstats/?q={}".format(obj.pk)
+    return format_html(f'''<a class="button" href="{path}">stats</a>''')
+
+  run.short_description = 'Run'
+  run.allow_tags = True
+  view_stats.short_description = 'Stats'
+  view_stats.allow_tags = True
+
+  list_display = ("job_name", "start_date", "end_date", "no_of_blogs", "start_no", "created_date", "run", "view_stats")
+  list_filter = ("job_name", "start_date")
+  readonly_fields = ("created_date",)
+
+class DjJobStats(admin.ModelAdmin):
+  def view_logs(self, obj):
+    path = "../joblogs/?q={}".format(obj.pk)
+    return format_html(f'''<a class="button" href="{path}">Logs</a>''')
+
+  view_logs.short_description = 'Stats'
+  view_logs.allow_tags = True
+  list_display = ("job", "status", "view_logs", "total_blogs", "no_of_blogs_extracted", "start_date", "end_date")
+  search_fields = ('job__pk',)
+
+class DjJobLogs(admin.ModelAdmin):
+  list_display = ("date", "log", "function_name")
+  search_fields = ('job_stats__pk',)
+
+
+# Register your models here.
+admin.site.register(Blog, DjBlogAdmin)
+admin.site.register(Students, DjStudentAdmin)
+admin.site.register(Job, DjJob)
+admin.site.register(JobStats, DjJobStats)
+admin.site.register(JobLogs, DjJobLogs)

Workshop4- Homework/myworld/members/apps.py

@@ -0,0 +1,90 @@
+from django.apps import AppConfig
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+import datetime
+from dateutil.parser import parse
+
+
+db_name = 'member_db'
+db_user = 'postgres'
+db_pass = '123456'
+db_host = 'psql-db'
+db_port = '5432'
+
+conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port)
+
+def add_row_to_blog(title, author, date, time):
+    sql = """INSERT INTO members_blog (title, release_date, blog_time, author, created_date) VALUES (%s, %s::DATE, %s::TIME, %s, NOW())"""
+
+    with conn:
+        with conn.cursor() as curs:
+            time=time.replace('\u202f',"")
+            curs.execute(sql, (title, date, time, author))
+
+def truncate_table():
+    print("Truncating contents all the tables")
+    with conn:
+        with conn.cursor() as curs:
+            curs.execute("TRUNCATE members_blog CASCADE;")
+
+def start_extraction(start_date=None, end_date=None):
+    print("Extraction started")
+    url = "https://blog.python.org/"
+
+    data = requests.get(url)
+    page_soup = BeautifulSoup(data.text, 'html.parser')
+
+    if start_date:
+        start_date = parse(start_date)
+    if end_date:
+        end_date = parse(end_date)
+
+    blogs = page_soup.select('div.date-outer')
+    truncate_table()
+    for blog in blogs:
+        date = blog.select('.date-header span')[0].get_text()
+
+        converted_date = parse(date)
+
+        if start_date and converted_date < start_date:
+            continue
+        if end_date and converted_date > end_date:
+            continue
+
+        post = blog.select('.post')[0]
+
+        title = ""
+        title_bar = post.select('.post-title')
+        if len(title_bar) > 0:
+            title = title_bar[0].text
+        else:
+            title = post.select('.post-body')[0].contents[0].text
+
+        # getting the author and blog time
+        post_footer = post.select('.post-footer')[0]
+
+        author = post_footer.select('.post-author span')[0].text
+
+        time = post_footer.select('abbr')[0].text
+
+        add_row_to_blog(title, author, date, time)
+
+        print("\nTitle:", title.strip('\n'))
+        print("Date:", date, )
+        print("Time:", time)
+        print("Author:", author)
+
+        # print("Number of blogs read:", count)
+        print(
+            "\n---------------------------------------------------------------------------------------------------------------\n")
+
+if __name__ == "__main__":
+    start_extraction()
+
+
+class MembersConfig(AppConfig):
+    default_auto_field='django.db.models.BigAutoField'
+    name = 'members'
+

Workshop4- Homework/myworld/members/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2 on 2023-05-23 05:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Members',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('firstname', models.CharField(max_length=255)),
+                ('lastname', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

Workshop4- Homework/myworld/members/migrations/0002_blog_job_joblogs_jobstats_students_delete_members_and_more.py

@@ -0,0 +1,78 @@
+# Generated by Django 4.2.5 on 2023-09-26 07:44
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Blog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=500)),
+                ('release_date', models.DateTimeField(verbose_name='Realse Date')),
+                ('blog_time', models.CharField(max_length=50)),
+                ('author', models.CharField(max_length=200)),
+                ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Created Date')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Job',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('job_name', models.CharField(max_length=500)),
+                ('start_date', models.DateTimeField(null=True, verbose_name='Blog start date')),
+                ('end_date', models.DateTimeField(null=True, verbose_name='Blog end date')),
+                ('start_no', models.IntegerField(null=True, verbose_name='No of blogs to skip')),
+                ('no_of_blogs', models.IntegerField(null=True, verbose_name='No of blogs to extract')),
+                ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Job created date')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='JobLogs',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('log', models.TextField(verbose_name='job logs')),
+                ('function_name', models.TextField(verbose_name='Function name')),
+                ('date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Log date')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='JobStats',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(max_length=50)),
+                ('total_blogs', models.IntegerField(null=True, verbose_name='Total blogs found')),
+                ('no_of_blogs_extracted', models.IntegerField(null=True, verbose_name='No of blogs extracted')),
+                ('start_date', models.DateTimeField(null=True, verbose_name='Extraction start date')),
+                ('end_date', models.DateTimeField(null=True, verbose_name='Extraction start date')),
+                ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='members.job')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Students',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('first_name', models.CharField(max_length=200)),
+                ('last_name', models.CharField(max_length=200)),
+                ('address', models.CharField(max_length=200)),
+                ('roll_number', models.IntegerField()),
+                ('mobile', models.CharField(max_length=10)),
+                ('branch', models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], max_length=10, null=True)),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='Members',
+        ),
+        migrations.AddField(
+            model_name='joblogs',
+            name='job_stats',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='members.jobstats'),
+        ),
+    ]