# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import re
import json
import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup
import collections

# +
import pymongo
import json
# from Processing.Processing import Processing

client = pymongo.MongoClient()
db = client["news"]
print(client['news'].list_collection_names())
coll_raw = db['bharian_raw']
coll_processed = db['thestar_pro']
# -

# coll_raw.delete_many({})
len(list(coll_raw.find()))

y = coll_raw.find_one()
y

list(x)

x = coll_raw.aggregate( 
    [
        {"$group":  { "_id": "$url", "count": { "$sum": 1 } } },
        {"$match": {"count": { "$gte": 2 }}}
    ]
)

x = coll_raw.update_many(
    {},
    {"$unset": { "processed_date": "" }}
)

urls = [y['_id'] for y in list(x)]
len(urls)

# +
# for url in urls:
#     x = coll_raw.delete_one({'url':url})
#     print(x.deleted_count)

# +
from datetime import datetime, timedelta
x= datetime.now() - timedelta(days=2)

#         news_date = datetime.datetime.strptime(date_1, "%d %B %Y @ %I:%M %p")
# -

len(list(coll_processed.find()))

list(coll_processed.find({'news_date':{'$gte':x}}))



# +
# ec_conn.indices.delete(index='news2')
# ec_conn.indices.create(index='news2', ignore=400, body=mapping)

# +
from datetime import datetime
from elasticsearch import Elasticsearch

INDEX_NAME = 'all_news'
ec_conn= Elasticsearch('http://127.0.0.1:9200')
ec_conn

ec_conn.search(index='all_news', body = {'from':0, 'size':1})['hits']['hits'][0]['_source'].keys()

# +

# ec.indices.create(index='test-index', ignore=400)
# ec_conn.indices.delete(index='news2')
# -

ec_conn.search(index='news', body = {'from':0, 'size':30})

import docker
client = docker.from_env()
for container in client.containers.list():
    print(container.id)
#     container.stop()

ec_conn.search(
    index="all_news", 
    body={
        'size':1,
        "query": {
             "multi_match": {"fields": ["title","content_text"], "query": 'covid malaysia kematian'}
        }
    }
)

ec_conn.search(
    index="all_news", 
    body={
        'size':1,
        "query": {
            "range": {
                "news_date": {
                    "lt": datetime.now()
                }
            }
        }
    }
)

# +
datequery=[
    {"index":"all_news"},
    {
        "body":{
            "query": {
                "range": {
                    "news_date": {
                        "lt": datetime.now()
                    }
                }
            }
        }
    }
]

textquery=[
    {"index":"all_news"},
    {
        "body":{
            'size':1,
            "query": {
                 "multi_match": {"fields": ["title","content_text"], "query": 'covid malaysia kematian'}
            }
        }
    }
]
ec_conn.msearch([datequery,textquery])

# -

query = {
    "bool":{  
         "filter":[  
             {
                "multi_match": {"fields": ["title","content_text"], "query": 'sabar'}
             },
            {  
               "terms":{  
                  "category":[  
                     "News",
                  ]
               }
            },
            {  
               "terms":{  
                  "topic":[  
                     "COVID-19",
                  ]
               }
            },
            {
                "range": {
                    "news_date": {"gte": datetime.datetime(2020,4,10), "lte": datetime.datetime(2020,4,12)},
                },
            }
         ]
    }
}

parser.parse(date)

import dateutil.parser as parser
parser.parse(r[0].replace('|',''))

response = requests.get('https://www.bharian.com.my/search?s=covid')
soup = BeautifulSoup(response.text, 'lxml')

[x.find('a').get('href') for x in (soup.find('div', class_='region-content').find_all('div', class_='block-content')[0].find_all('div', class_='views-field-title'))]

import re
r= re.search('\d.*', date)
r[0].replace('|','')

bahasa_months_dict = {
    "Januari": "January",
    "Februari": "February",
    "Mac": "March",
    "Mei": "May",
    "Julai": "July",
    "Ogos": "August",
    "Oktober": "October",
    "Disember": "December"
}
mon =[key for key in bahasa_months_dict.keys() if date.find(key) !=-1]
date = date.replace(mon[0], bahasa_months_dict[mon[0]]) if (len(mon) > 0) else date

cond = len([tar.find(key) for key in bahasa_months_dict.keys() if tar.find(key) !=-1])>0

mon

datetime.strptime('12 Apr 2020', "%d %b %Y")

tar = r[0].replace('|','')

date = soup.find('div', class_='node-meta').text
date

soup = BeautifulSoup(y['content_html'], 'html.parser')

x.find('div class="region region-content-bottom">')

soup.find('img').get('alt')

if soup.find('img') is not None:
    print('asda')

soup.find("img").get("data-src")



StarcoderdataJupyterScriptsDedupFiltered