unaiquenet - Backend - SOURCE CODE

# -*- coding: utf-8 -*-
#!/usr/bin/env python3
"""
Copyright (c) 2023, Sebastian Enger, M.Sc.
All rights reserved.

This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree (BSD-4-Clause). 

Frontend and Backend Source Code for Project:
- https://www.artikelschreiber.com/
- https://www.artikelschreiben.com/
- https://www.unaique.net/
"""
import os
import codecs
import datetime
from pathlib import Path
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=Warning)


from pydantic import BaseModel	#	pip3 install -U fastapi
import uvicorn	#	pip3 install -U uvicorn gunicorn
from fastapi import FastAPI, Form	#	pip3 install -U fastapi
#from starlette.responses import JSONResponse
#from starlette.applications import Starlette
#from starlette.middleware.wsgi import WSGIMiddleware

import sys
#sys.path.append('/home/unaique/library')
#import Mainify as mainify

sys.path.append('/home/unaique/library3')
#import Sublinify as subify
from Sublinify import Suble
mySubl = Suble()

version = f"{sys.version_info.major}.{sys.version_info.minor}"
#pip3 install -U gunicorn
# gunicorn -w 4 -k uvicorn.workers.UvicornWorker --reload True app:app --bind 127.0.0.1:8000
#pip3 install python-multipart
#uvicorn main:app --host 0.0.0.0 --port 80
#uvicorn /home/unaique/FlaskApp/app:app --reload
#pip3 install fastapi uvicorn
app 		= FastAPI()
#app = Starlette()  # will fastapi app work here?
#app.add_middleware(WSGIMiddleware)
import time

... [truncated, 166 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/python3.5 -S -W ignore
"""
Copyright (c) 2023, Sebastian Enger, M.Sc.
All rights reserved.

This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree (BSD-4-Clause). 

Frontend and Backend Source Code for Project:
- https://www.artikelschreiber.com/
- https://www.artikelschreiben.com/
- https://www.unaique.net/
"""
import re

import sys
sys.path.append('/home/unaique/library3')

import os
os.environ['TRANSFORMERS_CACHE'] 	= '/home/unaique/transformers_models_v3'
os.environ["MODEL_DIR"] 			= '/home/unaique/transformers_models_v3'
#os.environ['TRANSFORMERS_CACHE'] 	= '/dev/shm/'
#os.environ["MODEL_DIR"] 			= '/dev/shm'
os.environ['TF_CPP_MIN_LOG_LEVEL'] 	= '3'  # or any {'0', '1', '2'}
os.environ['TOKENIZERS_PARALLELISM']= 'true'

import openai	# https://github.com/openai/openai-python            pip install --upgrade openai
openai.api_key 						= "##################"

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

import torch	# # pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu # https://pytorch.org/get-started/locally/
import random
from PIL import Image, ImageDraw, ImageFont

#import fasttext # pip3 install -U fasttext
import numpy as np
#from fasttext import load_model
#from fasttext import util
from scipy.spatial.distance import cosine
from numpy import dot
from numpy.linalg import norm

# START: Google knowledge_graph processing
import pandas as pd
import numpy as np
#import advertools # pip3 install advertools
#from advertools import knowledge_graph

... [truncated, 609 more lines] ...

"""
Copyright (c) 2023, Sebastian Enger, M.Sc.
All rights reserved.

This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree (BSD-4-Clause). 

Frontend and Backend Source Code for Project:
- https://www.artikelschreiber.com/
- https://www.artikelschreiben.com/
- https://www.unaique.net/
"""

import logging
logging.getLogger().disabled = True
logging.disable(logging.WARNING)
logging.disable(logging.INFO)

import urllib.parse

import sys
sys.path.append('/home/unaique/library3')
#sys.path.append('/home/unaique/library')

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

"""
Fix SSL Problems:
# https://gankrin.org/how-to-fix-ssl-certificate_verify_failed-error-error-in-python/
$ sudo update-ca-certificates --fresh
$ export SSL_CERT_DIR=/etc/ssl/certs
"""

# pip3 install -U aiohttp asyncio
# https://www.twilio.com/blog/-asynchrone-http-anforderungen-python-aiohttp-asyncio
#import aiohttp
#import asyncio
import requests	# pip3 install --upgrade requests

import bleach #https://pypi.org/project/bleach/

from dns import resolver,reversename

import json

import codecs

import string
from string import printable

... [truncated, 2987 more lines] ...

# -*- coding: utf-8 -*-
#!/usr/bin/env python3

"""
Copyright (c) 2023, Sebastian Enger, M.Sc.
All rights reserved.

This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree (BSD-4-Clause). 

Frontend and Backend Source Code for Project:
- https://www.artikelschreiber.com/
- https://www.artikelschreiben.com/
- https://www.unaique.net/
"""

# pip3 install --upgrade spacy tensorflow gensim sumy keras markovify google-api-python-client beautifulsoup4

# https://stackoverflow.com/questions/35569042/ssl-certificate-verify-failed-with-python3
import ssl
ssl._create_default_https_context = ssl._create_unverified_context

import gc
import warnings
warnings.filterwarnings("ignore")
import codecs
import logging
#logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
logging.getLogger().disabled = True

# add the handlers to the logger
import os
import time
import json
import stat
import datetime
from datetime import datetime as dTime
from datetime import date
import psutil	# pip3 install -U psutil https://www.geeksforgeeks.org/how-to-get-current-cpu-and-ram-usage-in-python/
import sqlite3 as sql3

from random import randint
from time import sleep


sql3_lite_db 	= "/home/unaique/library3/sqlite/nodouble.sqlite"
# https://www.pragmaticlinux.com/2020/12/monitor-cpu-and-ram-usage-in-python-with-psutil/

from bionic_reading import BionicReading	# Bionic reading: https://github.com/Jwuthri/Riffling


... [truncated, 630 more lines] ...

#!/usr/bin/env python3
"""
    html5validate
    ------
    HTML5Lib based HTML5 validation module.
    MIT Licence - (C) 2019 Daniel Fairhead

"""

import warnings
from collections import namedtuple
import re
from xml.dom import Node

import html5lib

# in case we do need to track elements, here are some objects to hold them in:
DocType = namedtuple('DocType', ('name', 'publicId', 'systemId'))
StartTag = namedtuple('StartTag', ('name', 'attributes'))
VoidTag = namedtuple('VoidTag', ('name', 'attributes', 'has_children'))
EndTag = namedtuple('EndTag', ('name'))
Entity = namedtuple('Entity', ('name',))

SpaceCharacters = namedtuple('SpaceCharacters', ('data'))
Characters = namedtuple('Characters', ('data'))
Comment = namedtuple('Comment', ('data'))

# Splits <whitespace><anything><whitespace> apart.
TEXT_MATCH = re.compile(r'(\s*)(\S?.*\S)(\s*)')

from html5lib.html5parser import ParseError

class HTML5Invalid(Exception):
    pass

#class ParseError(HTML5Invalid):
#    pass

#class LintError(HTML5Invalid):
#    pass

class ValidationException(HTML5Invalid):
    pass

class InvalidTag(ValidationException):
    pass

class EmptyPage(ValidationException):
    pass


... [truncated, 556 more lines] ...