wg-backend-django/wg_api/utils/html_parser.py

from bs4 import BeautifulSoup
from tabulate import tabulate
import csv
import json
import requests

# Replace with curl
# f = open("utils/.tmp_html","r")
def get_session():
    login_data = {'username':"infidel","password":"roadsidepicnic", 'rememberMe':False}
    headers = {'Content-type': 'application/json'}
    inf_s = requests.Session()
    response = inf_s.post('http://10.30.1.106:5000/login', json=login_data)
    #inf_s = response.cookies
    print("-"*20)
    print("Session")
    print(inf_s)
    print(response.status_code)
    print("-"*20)

    return inf_s

def get_logs():
    inf_s = get_session()
    # curl --location --request POST 'http://10.30.1.106:5000/login' \
    
    html_data = inf_s.get('http://10.30.1.106:5000/status')
    
    #print(html_data.text)
    
    html_data = BeautifulSoup(html_data.text, "lxml")
    
    #print(html_data)
    #print(html_data.body.prettify())
    
    table = html_data.find_all('table')[0]
    
    # print(table)
    raw_data = []
    
    for row in table.find_all('tr'):
        tmp_data = []
        
        for row_data in row.find_all('td'):
    
            tmp_data.append(row_data)
            
            # print(row_data.text)
    
            # if not row_data.text:
            #     print("-")
            # if row_data.has_attr('title'):
            #     print(row_data['title'])
            #     # print("Got title for you")
        raw_data.append(tmp_data)
        # print(tmp_data)
    
    prod_data = []
    for proc_data in raw_data:
    
        if ( len(proc_data) > 0 ):
            i = 1
            tmp_data = []
    
            # if not "DTS" in str(proc_data[0]):
            #     # print("String ", proc_data[0])
            #     continue
    
            for proc_val in proc_data:
    
                data = BeautifulSoup(str(proc_val), "lxml")
                data = data.find('td')
                
                if data.has_attr('title'):
                    # Handle ugly byte notation
                    data = str(data['title'])
                    data = data[:-6]
                    data = int(data)
    
                elif len(data.text) < 1:
                    data = "-"
    
                elif i == 5:
                    data = str(data.text)
                    print(data)
                    data = data[:5]+"***"+data[-5:]
                    #data = data+"***"
    
                elif i == 7:
                    data = str(data.text)
                    if len(data) > 29:
                        data = data[:-20]
                    else:
                        data = data[:-10]
                else:
                    data = data.text
                
                my_data = data
                # print(i,"->",my_data)
                tmp_data.append(my_data)
    
                i=i+1
            # print("*"*100)
            prod_data.append(tmp_data)
    
    # print(prod_data)
    
    # print(json.dumps(prod_data))
    
    # with open("mon_data.csv", "w") as f:
    # 
    #     write = csv.writer(f)
    # 
    #     write.writerows(prod_data)
    # 
    # with open("mon_data.json", "w") as f:
    # 
    #     json.dump(prod_data, f)

    # print(">>"*100)
    # print(prod_data)
    # print(">>"*100)

    final_data = []
    #list_display = ['wgName', 'wgMail', 'wgKey', 'wgDin', 'wgDout', 'wgStatus', 'wgTStamp']
    list_display = ['wgName', 'wgMail', 'wgAIP', 'wgEP', 'wgKey', 'wgDin', 'wgDout', 'wgStatus', 'wgTStamp']
    j=1
    for v in prod_data:
        
        fucker_dict={}
        i=0

        for x in v:
            #print(x)
            fucker_dict[list_display[i]]=x
            i=i+1
        fucker_dict["data_id"]=j
        j=j+1 
        final_data.append(fucker_dict)    

    #sorted_footballers_by_goals = sorted(footballers_goals.items(), key=lambda x:x[1])
    final_data = sorted(final_data, key=lambda x: (x['wgStatus'], x['wgTStamp']), reverse=True)
    #final_data = sorted(final_data.items(), key=labda x:x[1]) 
    

    # r = requests.post("http://127.0.0.1:8000/api/wgAPI/?format=api", json=final_data)
    # final_data = json.dumps(final_data)

    return final_data

def get_profiles():
    
    inf_s = get_session()
    html_data = inf_s.get('http://10.30.1.106:5000/api/clients?')

    html_data = html_data.json()
    # html_data = json.loads(html_data)

    list_display = ['wgName', 'wgMail', 'wgKey', 'wgDin', 'wgDout', 'wgStatus', 'wgTStamp']
    final_data = []
    i = 1

    for x in html_data:
        current_data = x["Client"]
        current_data["data_id"]=i
        final_data.append(current_data)
        i = i + 1

    # final_data = json.dumps(final_data)
    # print(final_data)

    final_data = sorted(final_data, key=lambda x: (x['enabled'], x['created_at']), reverse=True)

    return final_data

# get_profiles()