Added login api
This commit is contained in:
parent
263218ed9e
commit
d5d3401265
247
acer-env/bin/Activate.ps1
Normal file
247
acer-env/bin/Activate.ps1
Normal file
@ -0,0 +1,247 @@
|
|||||||
|
<#
|
||||||
|
.Synopsis
|
||||||
|
Activate a Python virtual environment for the current PowerShell session.
|
||||||
|
|
||||||
|
.Description
|
||||||
|
Pushes the python executable for a virtual environment to the front of the
|
||||||
|
$Env:PATH environment variable and sets the prompt to signify that you are
|
||||||
|
in a Python virtual environment. Makes use of the command line switches as
|
||||||
|
well as the `pyvenv.cfg` file values present in the virtual environment.
|
||||||
|
|
||||||
|
.Parameter VenvDir
|
||||||
|
Path to the directory that contains the virtual environment to activate. The
|
||||||
|
default value for this is the parent of the directory that the Activate.ps1
|
||||||
|
script is located within.
|
||||||
|
|
||||||
|
.Parameter Prompt
|
||||||
|
The prompt prefix to display when this virtual environment is activated. By
|
||||||
|
default, this prompt is the name of the virtual environment folder (VenvDir)
|
||||||
|
surrounded by parentheses and followed by a single space (ie. '(.venv) ').
|
||||||
|
|
||||||
|
.Example
|
||||||
|
Activate.ps1
|
||||||
|
Activates the Python virtual environment that contains the Activate.ps1 script.
|
||||||
|
|
||||||
|
.Example
|
||||||
|
Activate.ps1 -Verbose
|
||||||
|
Activates the Python virtual environment that contains the Activate.ps1 script,
|
||||||
|
and shows extra information about the activation as it executes.
|
||||||
|
|
||||||
|
.Example
|
||||||
|
Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
|
||||||
|
Activates the Python virtual environment located in the specified location.
|
||||||
|
|
||||||
|
.Example
|
||||||
|
Activate.ps1 -Prompt "MyPython"
|
||||||
|
Activates the Python virtual environment that contains the Activate.ps1 script,
|
||||||
|
and prefixes the current prompt with the specified string (surrounded in
|
||||||
|
parentheses) while the virtual environment is active.
|
||||||
|
|
||||||
|
.Notes
|
||||||
|
On Windows, it may be required to enable this Activate.ps1 script by setting the
|
||||||
|
execution policy for the user. You can do this by issuing the following PowerShell
|
||||||
|
command:
|
||||||
|
|
||||||
|
PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
|
||||||
|
|
||||||
|
For more information on Execution Policies:
|
||||||
|
https://go.microsoft.com/fwlink/?LinkID=135170
|
||||||
|
|
||||||
|
#>
|
||||||
|
Param(
|
||||||
|
[Parameter(Mandatory = $false)]
|
||||||
|
[String]
|
||||||
|
$VenvDir,
|
||||||
|
[Parameter(Mandatory = $false)]
|
||||||
|
[String]
|
||||||
|
$Prompt
|
||||||
|
)
|
||||||
|
|
||||||
|
<# Function declarations --------------------------------------------------- #>
|
||||||
|
|
||||||
|
<#
|
||||||
|
.Synopsis
|
||||||
|
Remove all shell session elements added by the Activate script, including the
|
||||||
|
addition of the virtual environment's Python executable from the beginning of
|
||||||
|
the PATH variable.
|
||||||
|
|
||||||
|
.Parameter NonDestructive
|
||||||
|
If present, do not remove this function from the global namespace for the
|
||||||
|
session.
|
||||||
|
|
||||||
|
#>
|
||||||
|
function global:deactivate ([switch]$NonDestructive) {
|
||||||
|
# Revert to original values
|
||||||
|
|
||||||
|
# The prior prompt:
|
||||||
|
if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
|
||||||
|
Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
|
||||||
|
Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
|
||||||
|
}
|
||||||
|
|
||||||
|
# The prior PYTHONHOME:
|
||||||
|
if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
|
||||||
|
Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
|
||||||
|
Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
|
||||||
|
}
|
||||||
|
|
||||||
|
# The prior PATH:
|
||||||
|
if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
|
||||||
|
Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
|
||||||
|
Remove-Item -Path Env:_OLD_VIRTUAL_PATH
|
||||||
|
}
|
||||||
|
|
||||||
|
# Just remove the VIRTUAL_ENV altogether:
|
||||||
|
if (Test-Path -Path Env:VIRTUAL_ENV) {
|
||||||
|
Remove-Item -Path env:VIRTUAL_ENV
|
||||||
|
}
|
||||||
|
|
||||||
|
# Just remove VIRTUAL_ENV_PROMPT altogether.
|
||||||
|
if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) {
|
||||||
|
Remove-Item -Path env:VIRTUAL_ENV_PROMPT
|
||||||
|
}
|
||||||
|
|
||||||
|
# Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
|
||||||
|
if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
|
||||||
|
Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
|
||||||
|
}
|
||||||
|
|
||||||
|
# Leave deactivate function in the global namespace if requested:
|
||||||
|
if (-not $NonDestructive) {
|
||||||
|
Remove-Item -Path function:deactivate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
<#
|
||||||
|
.Description
|
||||||
|
Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
|
||||||
|
given folder, and returns them in a map.
|
||||||
|
|
||||||
|
For each line in the pyvenv.cfg file, if that line can be parsed into exactly
|
||||||
|
two strings separated by `=` (with any amount of whitespace surrounding the =)
|
||||||
|
then it is considered a `key = value` line. The left hand string is the key,
|
||||||
|
the right hand is the value.
|
||||||
|
|
||||||
|
If the value starts with a `'` or a `"` then the first and last character is
|
||||||
|
stripped from the value before being captured.
|
||||||
|
|
||||||
|
.Parameter ConfigDir
|
||||||
|
Path to the directory that contains the `pyvenv.cfg` file.
|
||||||
|
#>
|
||||||
|
function Get-PyVenvConfig(
|
||||||
|
[String]
|
||||||
|
$ConfigDir
|
||||||
|
) {
|
||||||
|
Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
|
||||||
|
|
||||||
|
# Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
|
||||||
|
$pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
|
||||||
|
|
||||||
|
# An empty map will be returned if no config file is found.
|
||||||
|
$pyvenvConfig = @{ }
|
||||||
|
|
||||||
|
if ($pyvenvConfigPath) {
|
||||||
|
|
||||||
|
Write-Verbose "File exists, parse `key = value` lines"
|
||||||
|
$pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
|
||||||
|
|
||||||
|
$pyvenvConfigContent | ForEach-Object {
|
||||||
|
$keyval = $PSItem -split "\s*=\s*", 2
|
||||||
|
if ($keyval[0] -and $keyval[1]) {
|
||||||
|
$val = $keyval[1]
|
||||||
|
|
||||||
|
# Remove extraneous quotations around a string value.
|
||||||
|
if ("'""".Contains($val.Substring(0, 1))) {
|
||||||
|
$val = $val.Substring(1, $val.Length - 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
$pyvenvConfig[$keyval[0]] = $val
|
||||||
|
Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $pyvenvConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
<# Begin Activate script --------------------------------------------------- #>
|
||||||
|
|
||||||
|
# Determine the containing directory of this script
|
||||||
|
$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
|
||||||
|
$VenvExecDir = Get-Item -Path $VenvExecPath
|
||||||
|
|
||||||
|
Write-Verbose "Activation script is located in path: '$VenvExecPath'"
|
||||||
|
Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
|
||||||
|
Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
|
||||||
|
|
||||||
|
# Set values required in priority: CmdLine, ConfigFile, Default
|
||||||
|
# First, get the location of the virtual environment, it might not be
|
||||||
|
# VenvExecDir if specified on the command line.
|
||||||
|
if ($VenvDir) {
|
||||||
|
Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
|
||||||
|
$VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
|
||||||
|
Write-Verbose "VenvDir=$VenvDir"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Next, read the `pyvenv.cfg` file to determine any required value such
|
||||||
|
# as `prompt`.
|
||||||
|
$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
|
||||||
|
|
||||||
|
# Next, set the prompt from the command line, or the config file, or
|
||||||
|
# just use the name of the virtual environment folder.
|
||||||
|
if ($Prompt) {
|
||||||
|
Write-Verbose "Prompt specified as argument, using '$Prompt'"
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
|
||||||
|
if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
|
||||||
|
Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
|
||||||
|
$Prompt = $pyvenvCfg['prompt'];
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
|
||||||
|
Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
|
||||||
|
$Prompt = Split-Path -Path $venvDir -Leaf
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Verbose "Prompt = '$Prompt'"
|
||||||
|
Write-Verbose "VenvDir='$VenvDir'"
|
||||||
|
|
||||||
|
# Deactivate any currently active virtual environment, but leave the
|
||||||
|
# deactivate function in place.
|
||||||
|
deactivate -nondestructive
|
||||||
|
|
||||||
|
# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
|
||||||
|
# that there is an activated venv.
|
||||||
|
$env:VIRTUAL_ENV = $VenvDir
|
||||||
|
|
||||||
|
if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
|
||||||
|
|
||||||
|
Write-Verbose "Setting prompt to '$Prompt'"
|
||||||
|
|
||||||
|
# Set the prompt to include the env name
|
||||||
|
# Make sure _OLD_VIRTUAL_PROMPT is global
|
||||||
|
function global:_OLD_VIRTUAL_PROMPT { "" }
|
||||||
|
Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
|
||||||
|
New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
|
||||||
|
|
||||||
|
function global:prompt {
|
||||||
|
Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
|
||||||
|
_OLD_VIRTUAL_PROMPT
|
||||||
|
}
|
||||||
|
$env:VIRTUAL_ENV_PROMPT = $Prompt
|
||||||
|
}
|
||||||
|
|
||||||
|
# Clear PYTHONHOME
|
||||||
|
if (Test-Path -Path Env:PYTHONHOME) {
|
||||||
|
Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
|
||||||
|
Remove-Item -Path Env:PYTHONHOME
|
||||||
|
}
|
||||||
|
|
||||||
|
# Add the venv to the PATH
|
||||||
|
Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
|
||||||
|
$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"
|
69
acer-env/bin/activate
Normal file
69
acer-env/bin/activate
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
# This file must be used with "source bin/activate" *from bash*
|
||||||
|
# you cannot run it directly
|
||||||
|
|
||||||
|
deactivate () {
|
||||||
|
# reset old environment variables
|
||||||
|
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
|
||||||
|
PATH="${_OLD_VIRTUAL_PATH:-}"
|
||||||
|
export PATH
|
||||||
|
unset _OLD_VIRTUAL_PATH
|
||||||
|
fi
|
||||||
|
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
|
||||||
|
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
|
||||||
|
export PYTHONHOME
|
||||||
|
unset _OLD_VIRTUAL_PYTHONHOME
|
||||||
|
fi
|
||||||
|
|
||||||
|
# This should detect bash and zsh, which have a hash command that must
|
||||||
|
# be called to get it to forget past commands. Without forgetting
|
||||||
|
# past commands the $PATH changes we made may not be respected
|
||||||
|
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
|
||||||
|
hash -r 2> /dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
|
||||||
|
PS1="${_OLD_VIRTUAL_PS1:-}"
|
||||||
|
export PS1
|
||||||
|
unset _OLD_VIRTUAL_PS1
|
||||||
|
fi
|
||||||
|
|
||||||
|
unset VIRTUAL_ENV
|
||||||
|
unset VIRTUAL_ENV_PROMPT
|
||||||
|
if [ ! "${1:-}" = "nondestructive" ] ; then
|
||||||
|
# Self destruct!
|
||||||
|
unset -f deactivate
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# unset irrelevant variables
|
||||||
|
deactivate nondestructive
|
||||||
|
|
||||||
|
VIRTUAL_ENV="/home/infidel/Sync/Project/ocp-wg-backend/acer-env"
|
||||||
|
export VIRTUAL_ENV
|
||||||
|
|
||||||
|
_OLD_VIRTUAL_PATH="$PATH"
|
||||||
|
PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||||
|
export PATH
|
||||||
|
|
||||||
|
# unset PYTHONHOME if set
|
||||||
|
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
|
||||||
|
# could use `if (set -u; : $PYTHONHOME) ;` in bash
|
||||||
|
if [ -n "${PYTHONHOME:-}" ] ; then
|
||||||
|
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
|
||||||
|
unset PYTHONHOME
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
|
||||||
|
_OLD_VIRTUAL_PS1="${PS1:-}"
|
||||||
|
PS1="(acer-env) ${PS1:-}"
|
||||||
|
export PS1
|
||||||
|
VIRTUAL_ENV_PROMPT="(acer-env) "
|
||||||
|
export VIRTUAL_ENV_PROMPT
|
||||||
|
fi
|
||||||
|
|
||||||
|
# This should detect bash and zsh, which have a hash command that must
|
||||||
|
# be called to get it to forget past commands. Without forgetting
|
||||||
|
# past commands the $PATH changes we made may not be respected
|
||||||
|
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
|
||||||
|
hash -r 2> /dev/null
|
||||||
|
fi
|
26
acer-env/bin/activate.csh
Normal file
26
acer-env/bin/activate.csh
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# This file must be used with "source bin/activate.csh" *from csh*.
|
||||||
|
# You cannot run it directly.
|
||||||
|
# Created by Davide Di Blasi <davidedb@gmail.com>.
|
||||||
|
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
|
||||||
|
|
||||||
|
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate'
|
||||||
|
|
||||||
|
# Unset irrelevant variables.
|
||||||
|
deactivate nondestructive
|
||||||
|
|
||||||
|
setenv VIRTUAL_ENV "/home/infidel/Sync/Project/ocp-wg-backend/acer-env"
|
||||||
|
|
||||||
|
set _OLD_VIRTUAL_PATH="$PATH"
|
||||||
|
setenv PATH "$VIRTUAL_ENV/bin:$PATH"
|
||||||
|
|
||||||
|
|
||||||
|
set _OLD_VIRTUAL_PROMPT="$prompt"
|
||||||
|
|
||||||
|
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
|
||||||
|
set prompt = "(acer-env) $prompt"
|
||||||
|
setenv VIRTUAL_ENV_PROMPT "(acer-env) "
|
||||||
|
endif
|
||||||
|
|
||||||
|
alias pydoc python -m pydoc
|
||||||
|
|
||||||
|
rehash
|
66
acer-env/bin/activate.fish
Normal file
66
acer-env/bin/activate.fish
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
# This file must be used with "source <venv>/bin/activate.fish" *from fish*
|
||||||
|
# (https://fishshell.com/); you cannot run it directly.
|
||||||
|
|
||||||
|
function deactivate -d "Exit virtual environment and return to normal shell environment"
|
||||||
|
# reset old environment variables
|
||||||
|
if test -n "$_OLD_VIRTUAL_PATH"
|
||||||
|
set -gx PATH $_OLD_VIRTUAL_PATH
|
||||||
|
set -e _OLD_VIRTUAL_PATH
|
||||||
|
end
|
||||||
|
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
|
||||||
|
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
|
||||||
|
set -e _OLD_VIRTUAL_PYTHONHOME
|
||||||
|
end
|
||||||
|
|
||||||
|
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
|
||||||
|
functions -e fish_prompt
|
||||||
|
set -e _OLD_FISH_PROMPT_OVERRIDE
|
||||||
|
functions -c _old_fish_prompt fish_prompt
|
||||||
|
functions -e _old_fish_prompt
|
||||||
|
end
|
||||||
|
|
||||||
|
set -e VIRTUAL_ENV
|
||||||
|
set -e VIRTUAL_ENV_PROMPT
|
||||||
|
if test "$argv[1]" != "nondestructive"
|
||||||
|
# Self-destruct!
|
||||||
|
functions -e deactivate
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Unset irrelevant variables.
|
||||||
|
deactivate nondestructive
|
||||||
|
|
||||||
|
set -gx VIRTUAL_ENV "/home/infidel/Sync/Project/ocp-wg-backend/acer-env"
|
||||||
|
|
||||||
|
set -gx _OLD_VIRTUAL_PATH $PATH
|
||||||
|
set -gx PATH "$VIRTUAL_ENV/bin" $PATH
|
||||||
|
|
||||||
|
# Unset PYTHONHOME if set.
|
||||||
|
if set -q PYTHONHOME
|
||||||
|
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
|
||||||
|
set -e PYTHONHOME
|
||||||
|
end
|
||||||
|
|
||||||
|
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
|
||||||
|
# fish uses a function instead of an env var to generate the prompt.
|
||||||
|
|
||||||
|
# Save the current fish_prompt function as the function _old_fish_prompt.
|
||||||
|
functions -c fish_prompt _old_fish_prompt
|
||||||
|
|
||||||
|
# With the original prompt function renamed, we can override with our own.
|
||||||
|
function fish_prompt
|
||||||
|
# Save the return status of the last command.
|
||||||
|
set -l old_status $status
|
||||||
|
|
||||||
|
# Output the venv prompt; color taken from the blue of the Python logo.
|
||||||
|
printf "%s%s%s" (set_color 4B8BBE) "(acer-env) " (set_color normal)
|
||||||
|
|
||||||
|
# Restore the return status of the previous command.
|
||||||
|
echo "exit $old_status" | .
|
||||||
|
# Output the original/"old" prompt.
|
||||||
|
_old_fish_prompt
|
||||||
|
end
|
||||||
|
|
||||||
|
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
|
||||||
|
set -gx VIRTUAL_ENV_PROMPT "(acer-env) "
|
||||||
|
end
|
8
acer-env/bin/django-admin
Executable file
8
acer-env/bin/django-admin
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from django.core.management import execute_from_command_line
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(execute_from_command_line())
|
8
acer-env/bin/normalizer
Executable file
8
acer-env/bin/normalizer
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from charset_normalizer.cli.normalizer import cli_detect
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(cli_detect())
|
8
acer-env/bin/pip
Executable file
8
acer-env/bin/pip
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pip._internal.cli.main import main
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(main())
|
8
acer-env/bin/pip3
Executable file
8
acer-env/bin/pip3
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pip._internal.cli.main import main
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(main())
|
8
acer-env/bin/pip3.10
Executable file
8
acer-env/bin/pip3.10
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pip._internal.cli.main import main
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(main())
|
1
acer-env/bin/python
Symbolic link
1
acer-env/bin/python
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
python3
|
1
acer-env/bin/python3
Symbolic link
1
acer-env/bin/python3
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
/usr/bin/python3
|
1
acer-env/bin/python3.10
Symbolic link
1
acer-env/bin/python3.10
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
python3
|
8
acer-env/bin/sqlformat
Executable file
8
acer-env/bin/sqlformat
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from sqlparse.__main__ import main
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(main())
|
8
acer-env/bin/tabulate
Executable file
8
acer-env/bin/tabulate
Executable file
@ -0,0 +1,8 @@
|
|||||||
|
#!/home/infidel/Sync/Project/ocp-wg-backend/acer-env/bin/python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from tabulate import _main
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||||
|
sys.exit(_main())
|
1046
acer-env/lib/python3.10/site-packages/Django-4.1.2.dist-info/AUTHORS
Normal file
1046
acer-env/lib/python3.10/site-packages/Django-4.1.2.dist-info/AUTHORS
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1 @@
|
|||||||
|
pip
|
@ -0,0 +1,27 @@
|
|||||||
|
Copyright (c) Django Software Foundation and individual contributors.
|
||||||
|
All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer in the
|
||||||
|
documentation and/or other materials provided with the distribution.
|
||||||
|
|
||||||
|
3. Neither the name of Django nor the names of its contributors may be used
|
||||||
|
to endorse or promote products derived from this software without
|
||||||
|
specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||||
|
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||||
|
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||||
|
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||||
|
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
|
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||||
|
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
|
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
@ -0,0 +1,290 @@
|
|||||||
|
Django is licensed under the three-clause BSD license; see the file
|
||||||
|
LICENSE for details.
|
||||||
|
|
||||||
|
Django includes code from the Python standard library, which is licensed under
|
||||||
|
the Python license, a permissive open source license. The copyright and license
|
||||||
|
is included below for compliance with Python's terms.
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
Copyright (c) 2001-present Python Software Foundation; All Rights Reserved
|
||||||
|
|
||||||
|
A. HISTORY OF THE SOFTWARE
|
||||||
|
==========================
|
||||||
|
|
||||||
|
Python was created in the early 1990s by Guido van Rossum at Stichting
|
||||||
|
Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
|
||||||
|
as a successor of a language called ABC. Guido remains Python's
|
||||||
|
principal author, although it includes many contributions from others.
|
||||||
|
|
||||||
|
In 1995, Guido continued his work on Python at the Corporation for
|
||||||
|
National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
|
||||||
|
in Reston, Virginia where he released several versions of the
|
||||||
|
software.
|
||||||
|
|
||||||
|
In May 2000, Guido and the Python core development team moved to
|
||||||
|
BeOpen.com to form the BeOpen PythonLabs team. In October of the same
|
||||||
|
year, the PythonLabs team moved to Digital Creations, which became
|
||||||
|
Zope Corporation. In 2001, the Python Software Foundation (PSF, see
|
||||||
|
https://www.python.org/psf/) was formed, a non-profit organization
|
||||||
|
created specifically to own Python-related Intellectual Property.
|
||||||
|
Zope Corporation was a sponsoring member of the PSF.
|
||||||
|
|
||||||
|
All Python releases are Open Source (see http://www.opensource.org for
|
||||||
|
the Open Source Definition). Historically, most, but not all, Python
|
||||||
|
releases have also been GPL-compatible; the table below summarizes
|
||||||
|
the various releases.
|
||||||
|
|
||||||
|
Release Derived Year Owner GPL-
|
||||||
|
from compatible? (1)
|
||||||
|
|
||||||
|
0.9.0 thru 1.2 1991-1995 CWI yes
|
||||||
|
1.3 thru 1.5.2 1.2 1995-1999 CNRI yes
|
||||||
|
1.6 1.5.2 2000 CNRI no
|
||||||
|
2.0 1.6 2000 BeOpen.com no
|
||||||
|
1.6.1 1.6 2001 CNRI yes (2)
|
||||||
|
2.1 2.0+1.6.1 2001 PSF no
|
||||||
|
2.0.1 2.0+1.6.1 2001 PSF yes
|
||||||
|
2.1.1 2.1+2.0.1 2001 PSF yes
|
||||||
|
2.1.2 2.1.1 2002 PSF yes
|
||||||
|
2.1.3 2.1.2 2002 PSF yes
|
||||||
|
2.2 and above 2.1.1 2001-now PSF yes
|
||||||
|
|
||||||
|
Footnotes:
|
||||||
|
|
||||||
|
(1) GPL-compatible doesn't mean that we're distributing Python under
|
||||||
|
the GPL. All Python licenses, unlike the GPL, let you distribute
|
||||||
|
a modified version without making your changes open source. The
|
||||||
|
GPL-compatible licenses make it possible to combine Python with
|
||||||
|
other software that is released under the GPL; the others don't.
|
||||||
|
|
||||||
|
(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
|
||||||
|
because its license has a choice of law clause. According to
|
||||||
|
CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
|
||||||
|
is "not incompatible" with the GPL.
|
||||||
|
|
||||||
|
Thanks to the many outside volunteers who have worked under Guido's
|
||||||
|
direction to make these releases possible.
|
||||||
|
|
||||||
|
|
||||||
|
B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
|
||||||
|
===============================================================
|
||||||
|
|
||||||
|
Python software and documentation are licensed under the
|
||||||
|
Python Software Foundation License Version 2.
|
||||||
|
|
||||||
|
Starting with Python 3.8.6, examples, recipes, and other code in
|
||||||
|
the documentation are dual licensed under the PSF License Version 2
|
||||||
|
and the Zero-Clause BSD license.
|
||||||
|
|
||||||
|
Some software incorporated into Python is under different licenses.
|
||||||
|
The licenses are listed with code falling under that license.
|
||||||
|
|
||||||
|
|
||||||
|
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
||||||
|
--------------------------------------------
|
||||||
|
|
||||||
|
1. This LICENSE AGREEMENT is between the Python Software Foundation
|
||||||
|
("PSF"), and the Individual or Organization ("Licensee") accessing and
|
||||||
|
otherwise using this software ("Python") in source or binary form and
|
||||||
|
its associated documentation.
|
||||||
|
|
||||||
|
2. Subject to the terms and conditions of this License Agreement, PSF hereby
|
||||||
|
grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
|
||||||
|
analyze, test, perform and/or display publicly, prepare derivative works,
|
||||||
|
distribute, and otherwise use Python alone or in any derivative version,
|
||||||
|
provided, however, that PSF's License Agreement and PSF's notice of copyright,
|
||||||
|
i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
|
||||||
|
2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation;
|
||||||
|
All Rights Reserved" are retained in Python alone or in any derivative version
|
||||||
|
prepared by Licensee.
|
||||||
|
|
||||||
|
3. In the event Licensee prepares a derivative work that is based on
|
||||||
|
or incorporates Python or any part thereof, and wants to make
|
||||||
|
the derivative work available to others as provided herein, then
|
||||||
|
Licensee hereby agrees to include in any such work a brief summary of
|
||||||
|
the changes made to Python.
|
||||||
|
|
||||||
|
4. PSF is making Python available to Licensee on an "AS IS"
|
||||||
|
basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||||
|
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
|
||||||
|
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||||
|
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
|
||||||
|
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||||
|
|
||||||
|
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
|
||||||
|
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
|
||||||
|
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
|
||||||
|
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||||
|
|
||||||
|
6. This License Agreement will automatically terminate upon a material
|
||||||
|
breach of its terms and conditions.
|
||||||
|
|
||||||
|
7. Nothing in this License Agreement shall be deemed to create any
|
||||||
|
relationship of agency, partnership, or joint venture between PSF and
|
||||||
|
Licensee. This License Agreement does not grant permission to use PSF
|
||||||
|
trademarks or trade name in a trademark sense to endorse or promote
|
||||||
|
products or services of Licensee, or any third party.
|
||||||
|
|
||||||
|
8. By copying, installing or otherwise using Python, Licensee
|
||||||
|
agrees to be bound by the terms and conditions of this License
|
||||||
|
Agreement.
|
||||||
|
|
||||||
|
|
||||||
|
BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
|
||||||
|
-------------------------------------------
|
||||||
|
|
||||||
|
BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
|
||||||
|
|
||||||
|
1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
|
||||||
|
office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
|
||||||
|
Individual or Organization ("Licensee") accessing and otherwise using
|
||||||
|
this software in source or binary form and its associated
|
||||||
|
documentation ("the Software").
|
||||||
|
|
||||||
|
2. Subject to the terms and conditions of this BeOpen Python License
|
||||||
|
Agreement, BeOpen hereby grants Licensee a non-exclusive,
|
||||||
|
royalty-free, world-wide license to reproduce, analyze, test, perform
|
||||||
|
and/or display publicly, prepare derivative works, distribute, and
|
||||||
|
otherwise use the Software alone or in any derivative version,
|
||||||
|
provided, however, that the BeOpen Python License is retained in the
|
||||||
|
Software, alone or in any derivative version prepared by Licensee.
|
||||||
|
|
||||||
|
3. BeOpen is making the Software available to Licensee on an "AS IS"
|
||||||
|
basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||||
|
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
|
||||||
|
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||||
|
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
|
||||||
|
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||||
|
|
||||||
|
4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
|
||||||
|
SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
|
||||||
|
AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
|
||||||
|
DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||||
|
|
||||||
|
5. This License Agreement will automatically terminate upon a material
|
||||||
|
breach of its terms and conditions.
|
||||||
|
|
||||||
|
6. This License Agreement shall be governed by and interpreted in all
|
||||||
|
respects by the law of the State of California, excluding conflict of
|
||||||
|
law provisions. Nothing in this License Agreement shall be deemed to
|
||||||
|
create any relationship of agency, partnership, or joint venture
|
||||||
|
between BeOpen and Licensee. This License Agreement does not grant
|
||||||
|
permission to use BeOpen trademarks or trade names in a trademark
|
||||||
|
sense to endorse or promote products or services of Licensee, or any
|
||||||
|
third party. As an exception, the "BeOpen Python" logos available at
|
||||||
|
http://www.pythonlabs.com/logos.html may be used according to the
|
||||||
|
permissions granted on that web page.
|
||||||
|
|
||||||
|
7. By copying, installing or otherwise using the software, Licensee
|
||||||
|
agrees to be bound by the terms and conditions of this License
|
||||||
|
Agreement.
|
||||||
|
|
||||||
|
|
||||||
|
CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
|
||||||
|
---------------------------------------
|
||||||
|
|
||||||
|
1. This LICENSE AGREEMENT is between the Corporation for National
|
||||||
|
Research Initiatives, having an office at 1895 Preston White Drive,
|
||||||
|
Reston, VA 20191 ("CNRI"), and the Individual or Organization
|
||||||
|
("Licensee") accessing and otherwise using Python 1.6.1 software in
|
||||||
|
source or binary form and its associated documentation.
|
||||||
|
|
||||||
|
2. Subject to the terms and conditions of this License Agreement, CNRI
|
||||||
|
hereby grants Licensee a nonexclusive, royalty-free, world-wide
|
||||||
|
license to reproduce, analyze, test, perform and/or display publicly,
|
||||||
|
prepare derivative works, distribute, and otherwise use Python 1.6.1
|
||||||
|
alone or in any derivative version, provided, however, that CNRI's
|
||||||
|
License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
|
||||||
|
1995-2001 Corporation for National Research Initiatives; All Rights
|
||||||
|
Reserved" are retained in Python 1.6.1 alone or in any derivative
|
||||||
|
version prepared by Licensee. Alternately, in lieu of CNRI's License
|
||||||
|
Agreement, Licensee may substitute the following text (omitting the
|
||||||
|
quotes): "Python 1.6.1 is made available subject to the terms and
|
||||||
|
conditions in CNRI's License Agreement. This Agreement together with
|
||||||
|
Python 1.6.1 may be located on the internet using the following
|
||||||
|
unique, persistent identifier (known as a handle): 1895.22/1013. This
|
||||||
|
Agreement may also be obtained from a proxy server on the internet
|
||||||
|
using the following URL: http://hdl.handle.net/1895.22/1013".
|
||||||
|
|
||||||
|
3. In the event Licensee prepares a derivative work that is based on
|
||||||
|
or incorporates Python 1.6.1 or any part thereof, and wants to make
|
||||||
|
the derivative work available to others as provided herein, then
|
||||||
|
Licensee hereby agrees to include in any such work a brief summary of
|
||||||
|
the changes made to Python 1.6.1.
|
||||||
|
|
||||||
|
4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
|
||||||
|
basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
|
||||||
|
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
|
||||||
|
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
|
||||||
|
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
|
||||||
|
INFRINGE ANY THIRD PARTY RIGHTS.
|
||||||
|
|
||||||
|
5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
|
||||||
|
1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
|
||||||
|
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
|
||||||
|
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
|
||||||
|
|
||||||
|
6. This License Agreement will automatically terminate upon a material
|
||||||
|
breach of its terms and conditions.
|
||||||
|
|
||||||
|
7. This License Agreement shall be governed by the federal
|
||||||
|
intellectual property law of the United States, including without
|
||||||
|
limitation the federal copyright law, and, to the extent such
|
||||||
|
U.S. federal law does not apply, by the law of the Commonwealth of
|
||||||
|
Virginia, excluding Virginia's conflict of law provisions.
|
||||||
|
Notwithstanding the foregoing, with regard to derivative works based
|
||||||
|
on Python 1.6.1 that incorporate non-separable material that was
|
||||||
|
previously distributed under the GNU General Public License (GPL), the
|
||||||
|
law of the Commonwealth of Virginia shall govern this License
|
||||||
|
Agreement only as to issues arising under or with respect to
|
||||||
|
Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this
|
||||||
|
License Agreement shall be deemed to create any relationship of
|
||||||
|
agency, partnership, or joint venture between CNRI and Licensee. This
|
||||||
|
License Agreement does not grant permission to use CNRI trademarks or
|
||||||
|
trade name in a trademark sense to endorse or promote products or
|
||||||
|
services of Licensee, or any third party.
|
||||||
|
|
||||||
|
8. By clicking on the "ACCEPT" button where indicated, or by copying,
|
||||||
|
installing or otherwise using Python 1.6.1, Licensee agrees to be
|
||||||
|
bound by the terms and conditions of this License Agreement.
|
||||||
|
|
||||||
|
ACCEPT
|
||||||
|
|
||||||
|
|
||||||
|
CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
|
||||||
|
The Netherlands. All rights reserved.
|
||||||
|
|
||||||
|
Permission to use, copy, modify, and distribute this software and its
|
||||||
|
documentation for any purpose and without fee is hereby granted,
|
||||||
|
provided that the above copyright notice appear in all copies and that
|
||||||
|
both that copyright notice and this permission notice appear in
|
||||||
|
supporting documentation, and that the name of Stichting Mathematisch
|
||||||
|
Centrum or CWI not be used in advertising or publicity pertaining to
|
||||||
|
distribution of the software without specific, written prior
|
||||||
|
permission.
|
||||||
|
|
||||||
|
STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
|
||||||
|
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||||
|
FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
|
||||||
|
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
|
||||||
|
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
|
||||||
|
Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
purpose with or without fee is hereby granted.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
PERFORMANCE OF THIS SOFTWARE.
|
@ -0,0 +1,100 @@
|
|||||||
|
Metadata-Version: 2.1
|
||||||
|
Name: Django
|
||||||
|
Version: 4.1.2
|
||||||
|
Summary: A high-level Python web framework that encourages rapid development and clean, pragmatic design.
|
||||||
|
Home-page: https://www.djangoproject.com/
|
||||||
|
Author: Django Software Foundation
|
||||||
|
Author-email: foundation@djangoproject.com
|
||||||
|
License: BSD-3-Clause
|
||||||
|
Project-URL: Documentation, https://docs.djangoproject.com/
|
||||||
|
Project-URL: Release notes, https://docs.djangoproject.com/en/stable/releases/
|
||||||
|
Project-URL: Funding, https://www.djangoproject.com/fundraising/
|
||||||
|
Project-URL: Source, https://github.com/django/django
|
||||||
|
Project-URL: Tracker, https://code.djangoproject.com/
|
||||||
|
Platform: UNKNOWN
|
||||||
|
Classifier: Development Status :: 5 - Production/Stable
|
||||||
|
Classifier: Environment :: Web Environment
|
||||||
|
Classifier: Framework :: Django
|
||||||
|
Classifier: Intended Audience :: Developers
|
||||||
|
Classifier: License :: OSI Approved :: BSD License
|
||||||
|
Classifier: Operating System :: OS Independent
|
||||||
|
Classifier: Programming Language :: Python
|
||||||
|
Classifier: Programming Language :: Python :: 3
|
||||||
|
Classifier: Programming Language :: Python :: 3 :: Only
|
||||||
|
Classifier: Programming Language :: Python :: 3.8
|
||||||
|
Classifier: Programming Language :: Python :: 3.9
|
||||||
|
Classifier: Programming Language :: Python :: 3.10
|
||||||
|
Classifier: Topic :: Internet :: WWW/HTTP
|
||||||
|
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
||||||
|
Classifier: Topic :: Internet :: WWW/HTTP :: WSGI
|
||||||
|
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
||||||
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||||
|
Requires-Python: >=3.8
|
||||||
|
License-File: LICENSE
|
||||||
|
License-File: LICENSE.python
|
||||||
|
License-File: AUTHORS
|
||||||
|
Requires-Dist: asgiref (<4,>=3.5.2)
|
||||||
|
Requires-Dist: sqlparse (>=0.2.2)
|
||||||
|
Requires-Dist: backports.zoneinfo ; python_version < "3.9"
|
||||||
|
Requires-Dist: tzdata ; sys_platform == "win32"
|
||||||
|
Provides-Extra: argon2
|
||||||
|
Requires-Dist: argon2-cffi (>=19.1.0) ; extra == 'argon2'
|
||||||
|
Provides-Extra: bcrypt
|
||||||
|
Requires-Dist: bcrypt ; extra == 'bcrypt'
|
||||||
|
|
||||||
|
======
|
||||||
|
Django
|
||||||
|
======
|
||||||
|
|
||||||
|
Django is a high-level Python web framework that encourages rapid development
|
||||||
|
and clean, pragmatic design. Thanks for checking it out.
|
||||||
|
|
||||||
|
All documentation is in the "``docs``" directory and online at
|
||||||
|
https://docs.djangoproject.com/en/stable/. If you're just getting started,
|
||||||
|
here's how we recommend you read the docs:
|
||||||
|
|
||||||
|
* First, read ``docs/intro/install.txt`` for instructions on installing Django.
|
||||||
|
|
||||||
|
* Next, work through the tutorials in order (``docs/intro/tutorial01.txt``,
|
||||||
|
``docs/intro/tutorial02.txt``, etc.).
|
||||||
|
|
||||||
|
* If you want to set up an actual deployment server, read
|
||||||
|
``docs/howto/deployment/index.txt`` for instructions.
|
||||||
|
|
||||||
|
* You'll probably want to read through the topical guides (in ``docs/topics``)
|
||||||
|
next; from there you can jump to the HOWTOs (in ``docs/howto``) for specific
|
||||||
|
problems, and check out the reference (``docs/ref``) for gory details.
|
||||||
|
|
||||||
|
* See ``docs/README`` for instructions on building an HTML version of the docs.
|
||||||
|
|
||||||
|
Docs are updated rigorously. If you find any problems in the docs, or think
|
||||||
|
they should be clarified in any way, please take 30 seconds to fill out a
|
||||||
|
ticket here: https://code.djangoproject.com/newticket
|
||||||
|
|
||||||
|
To get more help:
|
||||||
|
|
||||||
|
* Join the ``#django`` channel on ``irc.libera.chat``. Lots of helpful people
|
||||||
|
hang out there. See https://web.libera.chat if you're new to IRC.
|
||||||
|
|
||||||
|
* Join the django-users mailing list, or read the archives, at
|
||||||
|
https://groups.google.com/group/django-users.
|
||||||
|
|
||||||
|
To contribute to Django:
|
||||||
|
|
||||||
|
* Check out https://docs.djangoproject.com/en/dev/internals/contributing/ for
|
||||||
|
information about getting involved.
|
||||||
|
|
||||||
|
To run Django's test suite:
|
||||||
|
|
||||||
|
* Follow the instructions in the "Unit tests" section of
|
||||||
|
``docs/internals/contributing/writing-code/unit-tests.txt``, published online at
|
||||||
|
https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/#running-the-unit-tests
|
||||||
|
|
||||||
|
Supporting the Development of Django
|
||||||
|
====================================
|
||||||
|
|
||||||
|
Django's development depends on your contributions.
|
||||||
|
|
||||||
|
If you depend on Django, remember to support the Django Software Foundation: https://www.djangoproject.com/fundraising/
|
||||||
|
|
||||||
|
|
4451
acer-env/lib/python3.10/site-packages/Django-4.1.2.dist-info/RECORD
Normal file
4451
acer-env/lib/python3.10/site-packages/Django-4.1.2.dist-info/RECORD
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,5 @@
|
|||||||
|
Wheel-Version: 1.0
|
||||||
|
Generator: bdist_wheel (0.37.0)
|
||||||
|
Root-Is-Purelib: true
|
||||||
|
Tag: py3-none-any
|
||||||
|
|
@ -0,0 +1,3 @@
|
|||||||
|
[console_scripts]
|
||||||
|
django-admin = django.core.management:execute_from_command_line
|
||||||
|
|
@ -0,0 +1 @@
|
|||||||
|
django
|
@ -0,0 +1,132 @@
|
|||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import importlib
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
|
||||||
|
is_pypy = '__pypy__' in sys.builtin_module_names
|
||||||
|
|
||||||
|
|
||||||
|
warnings.filterwarnings('ignore',
|
||||||
|
r'.+ distutils\b.+ deprecated',
|
||||||
|
DeprecationWarning)
|
||||||
|
|
||||||
|
|
||||||
|
def warn_distutils_present():
|
||||||
|
if 'distutils' not in sys.modules:
|
||||||
|
return
|
||||||
|
if is_pypy and sys.version_info < (3, 7):
|
||||||
|
# PyPy for 3.6 unconditionally imports distutils, so bypass the warning
|
||||||
|
# https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250
|
||||||
|
return
|
||||||
|
warnings.warn(
|
||||||
|
"Distutils was imported before Setuptools, but importing Setuptools "
|
||||||
|
"also replaces the `distutils` module in `sys.modules`. This may lead "
|
||||||
|
"to undesirable behaviors or errors. To avoid these issues, avoid "
|
||||||
|
"using distutils directly, ensure that setuptools is installed in the "
|
||||||
|
"traditional way (e.g. not an editable install), and/or make sure "
|
||||||
|
"that setuptools is always imported before distutils.")
|
||||||
|
|
||||||
|
|
||||||
|
def clear_distutils():
|
||||||
|
if 'distutils' not in sys.modules:
|
||||||
|
return
|
||||||
|
warnings.warn("Setuptools is replacing distutils.")
|
||||||
|
mods = [name for name in sys.modules if re.match(r'distutils\b', name)]
|
||||||
|
for name in mods:
|
||||||
|
del sys.modules[name]
|
||||||
|
|
||||||
|
|
||||||
|
def enabled():
|
||||||
|
"""
|
||||||
|
Allow selection of distutils by environment variable.
|
||||||
|
"""
|
||||||
|
which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib')
|
||||||
|
return which == 'local'
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_local_distutils():
|
||||||
|
clear_distutils()
|
||||||
|
|
||||||
|
# With the DistutilsMetaFinder in place,
|
||||||
|
# perform an import to cause distutils to be
|
||||||
|
# loaded from setuptools._distutils. Ref #2906.
|
||||||
|
add_shim()
|
||||||
|
importlib.import_module('distutils')
|
||||||
|
remove_shim()
|
||||||
|
|
||||||
|
# check that submodules load as expected
|
||||||
|
core = importlib.import_module('distutils.core')
|
||||||
|
assert '_distutils' in core.__file__, core.__file__
|
||||||
|
|
||||||
|
|
||||||
|
def do_override():
|
||||||
|
"""
|
||||||
|
Ensure that the local copy of distutils is preferred over stdlib.
|
||||||
|
|
||||||
|
See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
|
||||||
|
for more motivation.
|
||||||
|
"""
|
||||||
|
if enabled():
|
||||||
|
warn_distutils_present()
|
||||||
|
ensure_local_distutils()
|
||||||
|
|
||||||
|
|
||||||
|
class DistutilsMetaFinder:
|
||||||
|
def find_spec(self, fullname, path, target=None):
|
||||||
|
if path is not None:
|
||||||
|
return
|
||||||
|
|
||||||
|
method_name = 'spec_for_{fullname}'.format(**locals())
|
||||||
|
method = getattr(self, method_name, lambda: None)
|
||||||
|
return method()
|
||||||
|
|
||||||
|
def spec_for_distutils(self):
|
||||||
|
import importlib.abc
|
||||||
|
import importlib.util
|
||||||
|
|
||||||
|
class DistutilsLoader(importlib.abc.Loader):
|
||||||
|
|
||||||
|
def create_module(self, spec):
|
||||||
|
return importlib.import_module('setuptools._distutils')
|
||||||
|
|
||||||
|
def exec_module(self, module):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return importlib.util.spec_from_loader('distutils', DistutilsLoader())
|
||||||
|
|
||||||
|
def spec_for_pip(self):
|
||||||
|
"""
|
||||||
|
Ensure stdlib distutils when running under pip.
|
||||||
|
See pypa/pip#8761 for rationale.
|
||||||
|
"""
|
||||||
|
if self.pip_imported_during_build():
|
||||||
|
return
|
||||||
|
clear_distutils()
|
||||||
|
self.spec_for_distutils = lambda: None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def pip_imported_during_build():
|
||||||
|
"""
|
||||||
|
Detect if pip is being imported in a build script. Ref #2355.
|
||||||
|
"""
|
||||||
|
import traceback
|
||||||
|
return any(
|
||||||
|
frame.f_globals['__file__'].endswith('setup.py')
|
||||||
|
for frame, line in traceback.walk_stack(None)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DISTUTILS_FINDER = DistutilsMetaFinder()
|
||||||
|
|
||||||
|
|
||||||
|
def add_shim():
|
||||||
|
sys.meta_path.insert(0, DISTUTILS_FINDER)
|
||||||
|
|
||||||
|
|
||||||
|
def remove_shim():
|
||||||
|
try:
|
||||||
|
sys.meta_path.remove(DISTUTILS_FINDER)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
|||||||
|
__import__('_distutils_hack').do_override()
|
@ -0,0 +1 @@
|
|||||||
|
pip
|
@ -0,0 +1,27 @@
|
|||||||
|
Copyright (c) Django Software Foundation and individual contributors.
|
||||||
|
All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer in the
|
||||||
|
documentation and/or other materials provided with the distribution.
|
||||||
|
|
||||||
|
3. Neither the name of Django nor the names of its contributors may be used
|
||||||
|
to endorse or promote products derived from this software without
|
||||||
|
specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||||
|
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||||
|
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||||
|
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||||
|
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
|
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||||
|
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
|
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
@ -0,0 +1,245 @@
|
|||||||
|
Metadata-Version: 2.1
|
||||||
|
Name: asgiref
|
||||||
|
Version: 3.5.2
|
||||||
|
Summary: ASGI specs, helper code, and adapters
|
||||||
|
Home-page: https://github.com/django/asgiref/
|
||||||
|
Author: Django Software Foundation
|
||||||
|
Author-email: foundation@djangoproject.com
|
||||||
|
License: BSD
|
||||||
|
Project-URL: Documentation, https://asgi.readthedocs.io/
|
||||||
|
Project-URL: Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions
|
||||||
|
Project-URL: Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt
|
||||||
|
Classifier: Development Status :: 5 - Production/Stable
|
||||||
|
Classifier: Environment :: Web Environment
|
||||||
|
Classifier: Intended Audience :: Developers
|
||||||
|
Classifier: License :: OSI Approved :: BSD License
|
||||||
|
Classifier: Operating System :: OS Independent
|
||||||
|
Classifier: Programming Language :: Python
|
||||||
|
Classifier: Programming Language :: Python :: 3
|
||||||
|
Classifier: Programming Language :: Python :: 3 :: Only
|
||||||
|
Classifier: Programming Language :: Python :: 3.7
|
||||||
|
Classifier: Programming Language :: Python :: 3.8
|
||||||
|
Classifier: Programming Language :: Python :: 3.9
|
||||||
|
Classifier: Programming Language :: Python :: 3.10
|
||||||
|
Classifier: Topic :: Internet :: WWW/HTTP
|
||||||
|
Requires-Python: >=3.7
|
||||||
|
License-File: LICENSE
|
||||||
|
Requires-Dist: typing-extensions ; python_version < "3.8"
|
||||||
|
Provides-Extra: tests
|
||||||
|
Requires-Dist: pytest ; extra == 'tests'
|
||||||
|
Requires-Dist: pytest-asyncio ; extra == 'tests'
|
||||||
|
Requires-Dist: mypy (>=0.800) ; extra == 'tests'
|
||||||
|
|
||||||
|
asgiref
|
||||||
|
=======
|
||||||
|
|
||||||
|
.. image:: https://api.travis-ci.org/django/asgiref.svg
|
||||||
|
:target: https://travis-ci.org/django/asgiref
|
||||||
|
|
||||||
|
.. image:: https://img.shields.io/pypi/v/asgiref.svg
|
||||||
|
:target: https://pypi.python.org/pypi/asgiref
|
||||||
|
|
||||||
|
ASGI is a standard for Python asynchronous web apps and servers to communicate
|
||||||
|
with each other, and positioned as an asynchronous successor to WSGI. You can
|
||||||
|
read more at https://asgi.readthedocs.io/en/latest/
|
||||||
|
|
||||||
|
This package includes ASGI base libraries, such as:
|
||||||
|
|
||||||
|
* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``
|
||||||
|
* Server base classes, ``asgiref.server``
|
||||||
|
* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``
|
||||||
|
|
||||||
|
|
||||||
|
Function wrappers
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
These allow you to wrap or decorate async or sync functions to call them from
|
||||||
|
the other style (so you can call async functions from a synchronous thread,
|
||||||
|
or vice-versa).
|
||||||
|
|
||||||
|
In particular:
|
||||||
|
|
||||||
|
* AsyncToSync lets a synchronous subthread stop and wait while the async
|
||||||
|
function is called on the main thread's event loop, and then control is
|
||||||
|
returned to the thread when the async function is finished.
|
||||||
|
|
||||||
|
* SyncToAsync lets async code call a synchronous function, which is run in
|
||||||
|
a threadpool and control returned to the async coroutine when the synchronous
|
||||||
|
function completes.
|
||||||
|
|
||||||
|
The idea is to make it easier to call synchronous APIs from async code and
|
||||||
|
asynchronous APIs from synchronous code so it's easier to transition code from
|
||||||
|
one style to the other. In the case of Channels, we wrap the (synchronous)
|
||||||
|
Django view system with SyncToAsync to allow it to run inside the (asynchronous)
|
||||||
|
ASGI server.
|
||||||
|
|
||||||
|
Note that exactly what threads things run in is very specific, and aimed to
|
||||||
|
keep maximum compatibility with old synchronous code. See
|
||||||
|
"Synchronous code & Threads" below for a full explanation. By default,
|
||||||
|
``sync_to_async`` will run all synchronous code in the program in the same
|
||||||
|
thread for safety reasons; you can disable this for more performance with
|
||||||
|
``@sync_to_async(thread_sensitive=False)``, but make sure that your code does
|
||||||
|
not rely on anything bound to threads (like database connections) when you do.
|
||||||
|
|
||||||
|
|
||||||
|
Threadlocal replacement
|
||||||
|
-----------------------
|
||||||
|
|
||||||
|
This is a drop-in replacement for ``threading.local`` that works with both
|
||||||
|
threads and asyncio Tasks. Even better, it will proxy values through from a
|
||||||
|
task-local context to a thread-local context when you use ``sync_to_async``
|
||||||
|
to run things in a threadpool, and vice-versa for ``async_to_sync``.
|
||||||
|
|
||||||
|
If you instead want true thread- and task-safety, you can set
|
||||||
|
``thread_critical`` on the Local object to ensure this instead.
|
||||||
|
|
||||||
|
|
||||||
|
Server base classes
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Includes a ``StatelessServer`` class which provides all the hard work of
|
||||||
|
writing a stateless server (as in, does not handle direct incoming sockets
|
||||||
|
but instead consumes external streams or sockets to work out what is happening).
|
||||||
|
|
||||||
|
An example of such a server would be a chatbot server that connects out to
|
||||||
|
a central chat server and provides a "connection scope" per user chatting to
|
||||||
|
it. There's only one actual connection, but the server has to separate things
|
||||||
|
into several scopes for easier writing of the code.
|
||||||
|
|
||||||
|
You can see an example of this being used in `frequensgi <https://github.com/andrewgodwin/frequensgi>`_.
|
||||||
|
|
||||||
|
|
||||||
|
WSGI-to-ASGI adapter
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
Allows you to wrap a WSGI application so it appears as a valid ASGI application.
|
||||||
|
|
||||||
|
Simply wrap it around your WSGI application like so::
|
||||||
|
|
||||||
|
asgi_application = WsgiToAsgi(wsgi_application)
|
||||||
|
|
||||||
|
The WSGI application will be run in a synchronous threadpool, and the wrapped
|
||||||
|
ASGI application will be one that accepts ``http`` class messages.
|
||||||
|
|
||||||
|
Please note that not all extended features of WSGI may be supported (such as
|
||||||
|
file handles for incoming POST bodies).
|
||||||
|
|
||||||
|
|
||||||
|
Dependencies
|
||||||
|
------------
|
||||||
|
|
||||||
|
``asgiref`` requires Python 3.7 or higher.
|
||||||
|
|
||||||
|
|
||||||
|
Contributing
|
||||||
|
------------
|
||||||
|
|
||||||
|
Please refer to the
|
||||||
|
`main Channels contributing docs <https://github.com/django/channels/blob/master/CONTRIBUTING.rst>`_.
|
||||||
|
|
||||||
|
|
||||||
|
Testing
|
||||||
|
'''''''
|
||||||
|
|
||||||
|
To run tests, make sure you have installed the ``tests`` extra with the package::
|
||||||
|
|
||||||
|
cd asgiref/
|
||||||
|
pip install -e .[tests]
|
||||||
|
pytest
|
||||||
|
|
||||||
|
|
||||||
|
Building the documentation
|
||||||
|
''''''''''''''''''''''''''
|
||||||
|
|
||||||
|
The documentation uses `Sphinx <http://www.sphinx-doc.org>`_::
|
||||||
|
|
||||||
|
cd asgiref/docs/
|
||||||
|
pip install sphinx
|
||||||
|
|
||||||
|
To build the docs, you can use the default tools::
|
||||||
|
|
||||||
|
sphinx-build -b html . _build/html # or `make html`, if you've got make set up
|
||||||
|
cd _build/html
|
||||||
|
python -m http.server
|
||||||
|
|
||||||
|
...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload
|
||||||
|
your documentation changes automatically::
|
||||||
|
|
||||||
|
pip install sphinx-autobuild
|
||||||
|
sphinx-autobuild . _build/html
|
||||||
|
|
||||||
|
|
||||||
|
Releasing
|
||||||
|
'''''''''
|
||||||
|
|
||||||
|
To release, first add details to CHANGELOG.txt and update the version number in ``asgiref/__init__.py``.
|
||||||
|
|
||||||
|
Then, build and push the packages::
|
||||||
|
|
||||||
|
python -m build
|
||||||
|
twine upload dist/*
|
||||||
|
rm -r build/ dist/
|
||||||
|
|
||||||
|
|
||||||
|
Implementation Details
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
Synchronous code & threads
|
||||||
|
''''''''''''''''''''''''''
|
||||||
|
|
||||||
|
The ``asgiref.sync`` module provides two wrappers that let you go between
|
||||||
|
asynchronous and synchronous code at will, while taking care of the rough edges
|
||||||
|
for you.
|
||||||
|
|
||||||
|
Unfortunately, the rough edges are numerous, and the code has to work especially
|
||||||
|
hard to keep things in the same thread as much as possible. Notably, the
|
||||||
|
restrictions we are working with are:
|
||||||
|
|
||||||
|
* All synchronous code called through ``SyncToAsync`` and marked with
|
||||||
|
``thread_sensitive`` should run in the same thread as each other (and if the
|
||||||
|
outer layer of the program is synchronous, the main thread)
|
||||||
|
|
||||||
|
* If a thread already has a running async loop, ``AsyncToSync`` can't run things
|
||||||
|
on that loop if it's blocked on synchronous code that is above you in the
|
||||||
|
call stack.
|
||||||
|
|
||||||
|
The first compromise you get to might be that ``thread_sensitive`` code should
|
||||||
|
just run in the same thread and not spawn in a sub-thread, fulfilling the first
|
||||||
|
restriction, but that immediately runs you into the second restriction.
|
||||||
|
|
||||||
|
The only real solution is to essentially have a variant of ThreadPoolExecutor
|
||||||
|
that executes any ``thread_sensitive`` code on the outermost synchronous
|
||||||
|
thread - either the main thread, or a single spawned subthread.
|
||||||
|
|
||||||
|
This means you now have two basic states:
|
||||||
|
|
||||||
|
* If the outermost layer of your program is synchronous, then all async code
|
||||||
|
run through ``AsyncToSync`` will run in a per-call event loop in arbitrary
|
||||||
|
sub-threads, while all ``thread_sensitive`` code will run in the main thread.
|
||||||
|
|
||||||
|
* If the outermost layer of your program is asynchronous, then all async code
|
||||||
|
runs on the main thread's event loop, and all ``thread_sensitive`` synchronous
|
||||||
|
code will run in a single shared sub-thread.
|
||||||
|
|
||||||
|
Crucially, this means that in both cases there is a thread which is a shared
|
||||||
|
resource that all ``thread_sensitive`` code must run on, and there is a chance
|
||||||
|
that this thread is currently blocked on its own ``AsyncToSync`` call. Thus,
|
||||||
|
``AsyncToSync`` needs to act as an executor for thread code while it's blocking.
|
||||||
|
|
||||||
|
The ``CurrentThreadExecutor`` class provides this functionality; rather than
|
||||||
|
simply waiting on a Future, you can call its ``run_until_future`` method and
|
||||||
|
it will run submitted code until that Future is done. This means that code
|
||||||
|
inside the call can then run code on your thread.
|
||||||
|
|
||||||
|
|
||||||
|
Maintenance and Security
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
To report security issues, please contact security@djangoproject.com. For GPG
|
||||||
|
signatures and more security process information, see
|
||||||
|
https://docs.djangoproject.com/en/dev/internals/security/.
|
||||||
|
|
||||||
|
To report bugs or request new features, please open a new GitHub issue.
|
||||||
|
|
||||||
|
This repository is part of the Channels project. For the shepherd and maintenance team, please see the
|
||||||
|
`main Channels readme <https://github.com/django/channels/blob/master/README.rst>`_.
|
@ -0,0 +1,28 @@
|
|||||||
|
asgiref-3.5.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||||
|
asgiref-3.5.2.dist-info/LICENSE,sha256=uEZBXRtRTpwd_xSiLeuQbXlLxUbKYSn5UKGM0JHipmk,1552
|
||||||
|
asgiref-3.5.2.dist-info/METADATA,sha256=3JU5Zw-j9qCKPcuf3cJZ5dVispB_b7UXU0fnQVp9DDA,9143
|
||||||
|
asgiref-3.5.2.dist-info/RECORD,,
|
||||||
|
asgiref-3.5.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||||
|
asgiref-3.5.2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
||||||
|
asgiref-3.5.2.dist-info/top_level.txt,sha256=bokQjCzwwERhdBiPdvYEZa4cHxT4NCeAffQNUqJ8ssg,8
|
||||||
|
asgiref/__init__.py,sha256=LtYJ5AVwuiAlsrJUQwzHZMrGMIRn7cuIoIt4OznYy6c,22
|
||||||
|
asgiref/__pycache__/__init__.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/compatibility.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/current_thread_executor.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/local.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/server.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/sync.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/testing.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/timeout.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/typing.cpython-310.pyc,,
|
||||||
|
asgiref/__pycache__/wsgi.cpython-310.pyc,,
|
||||||
|
asgiref/compatibility.py,sha256=MVH2bEdiCMMVTLbE-1V6KiU7q4LwqzP7PIufeXa-njM,1598
|
||||||
|
asgiref/current_thread_executor.py,sha256=oeH8zv2tTmcbpxdUmOSMzbEXzeY5nJzIMFvzprE95gA,2801
|
||||||
|
asgiref/local.py,sha256=nx5RqVFLYgUJVaxzApuQUW7dd9y21sruMYdgISoRs1k,4854
|
||||||
|
asgiref/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||||
|
asgiref/server.py,sha256=egTQhZo1k4G0F7SSBQNp_VOekpGcjBJZU2kkCoiGC_M,6005
|
||||||
|
asgiref/sync.py,sha256=3P813NHl3EHPMtzPEjaBelmjV_JUw97zYbtx-MmLUiw,20185
|
||||||
|
asgiref/testing.py,sha256=3byNRV7Oto_Fg8Z-fErQJ3yGf7OQlcUexbN_cDQugzQ,3119
|
||||||
|
asgiref/timeout.py,sha256=5Ekbmn3X1HPR55qgx-hPJMPEu_-YoivHqNhFEitiSYE,3440
|
||||||
|
asgiref/typing.py,sha256=MZ7vbJY1F7EQqo9gL9pMSFRMw9b_SQrQQsnvlJQ2iP4,5603
|
||||||
|
asgiref/wsgi.py,sha256=-L0eo_uK_dq7EPjv1meW1BRGytURaO9NPESxnJc9CtA,6575
|
@ -0,0 +1,5 @@
|
|||||||
|
Wheel-Version: 1.0
|
||||||
|
Generator: bdist_wheel (0.37.1)
|
||||||
|
Root-Is-Purelib: true
|
||||||
|
Tag: py3-none-any
|
||||||
|
|
@ -0,0 +1 @@
|
|||||||
|
asgiref
|
@ -0,0 +1 @@
|
|||||||
|
__version__ = "3.5.2"
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,47 @@
|
|||||||
|
import asyncio
|
||||||
|
import inspect
|
||||||
|
|
||||||
|
|
||||||
|
def is_double_callable(application):
|
||||||
|
"""
|
||||||
|
Tests to see if an application is a legacy-style (double-callable) application.
|
||||||
|
"""
|
||||||
|
# Look for a hint on the object first
|
||||||
|
if getattr(application, "_asgi_single_callable", False):
|
||||||
|
return False
|
||||||
|
if getattr(application, "_asgi_double_callable", False):
|
||||||
|
return True
|
||||||
|
# Uninstanted classes are double-callable
|
||||||
|
if inspect.isclass(application):
|
||||||
|
return True
|
||||||
|
# Instanted classes depend on their __call__
|
||||||
|
if hasattr(application, "__call__"):
|
||||||
|
# We only check to see if its __call__ is a coroutine function -
|
||||||
|
# if it's not, it still might be a coroutine function itself.
|
||||||
|
if asyncio.iscoroutinefunction(application.__call__):
|
||||||
|
return False
|
||||||
|
# Non-classes we just check directly
|
||||||
|
return not asyncio.iscoroutinefunction(application)
|
||||||
|
|
||||||
|
|
||||||
|
def double_to_single_callable(application):
|
||||||
|
"""
|
||||||
|
Transforms a double-callable ASGI application into a single-callable one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def new_application(scope, receive, send):
|
||||||
|
instance = application(scope)
|
||||||
|
return await instance(receive, send)
|
||||||
|
|
||||||
|
return new_application
|
||||||
|
|
||||||
|
|
||||||
|
def guarantee_single_callable(application):
|
||||||
|
"""
|
||||||
|
Takes either a single- or double-callable application and always returns it
|
||||||
|
in single-callable style. Use this to add backwards compatibility for ASGI
|
||||||
|
2.0 applications to your server/test harness/etc.
|
||||||
|
"""
|
||||||
|
if is_double_callable(application):
|
||||||
|
application = double_to_single_callable(application)
|
||||||
|
return application
|
@ -0,0 +1,81 @@
|
|||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import Executor, Future
|
||||||
|
|
||||||
|
|
||||||
|
class _WorkItem:
|
||||||
|
"""
|
||||||
|
Represents an item needing to be run in the executor.
|
||||||
|
Copied from ThreadPoolExecutor (but it's private, so we're not going to rely on importing it)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, future, fn, args, kwargs):
|
||||||
|
self.future = future
|
||||||
|
self.fn = fn
|
||||||
|
self.args = args
|
||||||
|
self.kwargs = kwargs
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
if not self.future.set_running_or_notify_cancel():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
result = self.fn(*self.args, **self.kwargs)
|
||||||
|
except BaseException as exc:
|
||||||
|
self.future.set_exception(exc)
|
||||||
|
# Break a reference cycle with the exception 'exc'
|
||||||
|
self = None
|
||||||
|
else:
|
||||||
|
self.future.set_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
class CurrentThreadExecutor(Executor):
|
||||||
|
"""
|
||||||
|
An Executor that actually runs code in the thread it is instantiated in.
|
||||||
|
Passed to other threads running async code, so they can run sync code in
|
||||||
|
the thread they came from.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._work_thread = threading.current_thread()
|
||||||
|
self._work_queue = queue.Queue()
|
||||||
|
self._broken = False
|
||||||
|
|
||||||
|
def run_until_future(self, future):
|
||||||
|
"""
|
||||||
|
Runs the code in the work queue until a result is available from the future.
|
||||||
|
Should be run from the thread the executor is initialised in.
|
||||||
|
"""
|
||||||
|
# Check we're in the right thread
|
||||||
|
if threading.current_thread() != self._work_thread:
|
||||||
|
raise RuntimeError(
|
||||||
|
"You cannot run CurrentThreadExecutor from a different thread"
|
||||||
|
)
|
||||||
|
future.add_done_callback(self._work_queue.put)
|
||||||
|
# Keep getting and running work items until we get the future we're waiting for
|
||||||
|
# back via the future's done callback.
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
# Get a work item and run it
|
||||||
|
work_item = self._work_queue.get()
|
||||||
|
if work_item is future:
|
||||||
|
return
|
||||||
|
work_item.run()
|
||||||
|
del work_item
|
||||||
|
finally:
|
||||||
|
self._broken = True
|
||||||
|
|
||||||
|
def submit(self, fn, *args, **kwargs):
|
||||||
|
# Check they're not submitting from the same thread
|
||||||
|
if threading.current_thread() == self._work_thread:
|
||||||
|
raise RuntimeError(
|
||||||
|
"You cannot submit onto CurrentThreadExecutor from its own thread"
|
||||||
|
)
|
||||||
|
# Check they're not too late or the executor errored
|
||||||
|
if self._broken:
|
||||||
|
raise RuntimeError("CurrentThreadExecutor already quit or is broken")
|
||||||
|
# Add to work queue
|
||||||
|
f = Future()
|
||||||
|
work_item = _WorkItem(f, fn, args, kwargs)
|
||||||
|
self._work_queue.put(work_item)
|
||||||
|
# Return the future
|
||||||
|
return f
|
120
acer-env/lib/python3.10/site-packages/asgiref/local.py
Normal file
120
acer-env/lib/python3.10/site-packages/asgiref/local.py
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
import random
|
||||||
|
import string
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import weakref
|
||||||
|
|
||||||
|
|
||||||
|
class Local:
|
||||||
|
"""
|
||||||
|
A drop-in replacement for threading.locals that also works with asyncio
|
||||||
|
Tasks (via the current_task asyncio method), and passes locals through
|
||||||
|
sync_to_async and async_to_sync.
|
||||||
|
|
||||||
|
Specifically:
|
||||||
|
- Locals work per-coroutine on any thread not spawned using asgiref
|
||||||
|
- Locals work per-thread on any thread not spawned using asgiref
|
||||||
|
- Locals are shared with the parent coroutine when using sync_to_async
|
||||||
|
- Locals are shared with the parent thread when using async_to_sync
|
||||||
|
(and if that thread was launched using sync_to_async, with its parent
|
||||||
|
coroutine as well, with this working for indefinite levels of nesting)
|
||||||
|
|
||||||
|
Set thread_critical to True to not allow locals to pass from an async Task
|
||||||
|
to a thread it spawns. This is needed for code that truly needs
|
||||||
|
thread-safety, as opposed to things used for helpful context (e.g. sqlite
|
||||||
|
does not like being called from a different thread to the one it is from).
|
||||||
|
Thread-critical code will still be differentiated per-Task within a thread
|
||||||
|
as it is expected it does not like concurrent access.
|
||||||
|
|
||||||
|
This doesn't use contextvars as it needs to support 3.6. Once it can support
|
||||||
|
3.7 only, we can then reimplement the storage more nicely.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, thread_critical: bool = False) -> None:
|
||||||
|
self._thread_critical = thread_critical
|
||||||
|
self._thread_lock = threading.RLock()
|
||||||
|
self._context_refs: "weakref.WeakSet[object]" = weakref.WeakSet()
|
||||||
|
# Random suffixes stop accidental reuse between different Locals,
|
||||||
|
# though we try to force deletion as well.
|
||||||
|
self._attr_name = "_asgiref_local_impl_{}_{}".format(
|
||||||
|
id(self),
|
||||||
|
"".join(random.choice(string.ascii_letters) for i in range(8)),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_context_id(self):
|
||||||
|
"""
|
||||||
|
Get the ID we should use for looking up variables
|
||||||
|
"""
|
||||||
|
# Prevent a circular reference
|
||||||
|
from .sync import AsyncToSync, SyncToAsync
|
||||||
|
|
||||||
|
# First, pull the current task if we can
|
||||||
|
context_id = SyncToAsync.get_current_task()
|
||||||
|
context_is_async = True
|
||||||
|
# OK, let's try for a thread ID
|
||||||
|
if context_id is None:
|
||||||
|
context_id = threading.current_thread()
|
||||||
|
context_is_async = False
|
||||||
|
# If we're thread-critical, we stop here, as we can't share contexts.
|
||||||
|
if self._thread_critical:
|
||||||
|
return context_id
|
||||||
|
# Now, take those and see if we can resolve them through the launch maps
|
||||||
|
for i in range(sys.getrecursionlimit()):
|
||||||
|
try:
|
||||||
|
if context_is_async:
|
||||||
|
# Tasks have a source thread in AsyncToSync
|
||||||
|
context_id = AsyncToSync.launch_map[context_id]
|
||||||
|
context_is_async = False
|
||||||
|
else:
|
||||||
|
# Threads have a source task in SyncToAsync
|
||||||
|
context_id = SyncToAsync.launch_map[context_id]
|
||||||
|
context_is_async = True
|
||||||
|
except KeyError:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# Catch infinite loops (they happen if you are screwing around
|
||||||
|
# with AsyncToSync implementations)
|
||||||
|
raise RuntimeError("Infinite launch_map loops")
|
||||||
|
return context_id
|
||||||
|
|
||||||
|
def _get_storage(self):
|
||||||
|
context_obj = self._get_context_id()
|
||||||
|
if not hasattr(context_obj, self._attr_name):
|
||||||
|
setattr(context_obj, self._attr_name, {})
|
||||||
|
self._context_refs.add(context_obj)
|
||||||
|
return getattr(context_obj, self._attr_name)
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
try:
|
||||||
|
for context_obj in self._context_refs:
|
||||||
|
try:
|
||||||
|
delattr(context_obj, self._attr_name)
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
except TypeError:
|
||||||
|
# WeakSet.__iter__ can crash when interpreter is shutting down due
|
||||||
|
# to _IterationGuard being None.
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __getattr__(self, key):
|
||||||
|
with self._thread_lock:
|
||||||
|
storage = self._get_storage()
|
||||||
|
if key in storage:
|
||||||
|
return storage[key]
|
||||||
|
else:
|
||||||
|
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
||||||
|
|
||||||
|
def __setattr__(self, key, value):
|
||||||
|
if key in ("_context_refs", "_thread_critical", "_thread_lock", "_attr_name"):
|
||||||
|
return super().__setattr__(key, value)
|
||||||
|
with self._thread_lock:
|
||||||
|
storage = self._get_storage()
|
||||||
|
storage[key] = value
|
||||||
|
|
||||||
|
def __delattr__(self, key):
|
||||||
|
with self._thread_lock:
|
||||||
|
storage = self._get_storage()
|
||||||
|
if key in storage:
|
||||||
|
del storage[key]
|
||||||
|
else:
|
||||||
|
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
157
acer-env/lib/python3.10/site-packages/asgiref/server.py
Normal file
157
acer-env/lib/python3.10/site-packages/asgiref/server.py
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
from .compatibility import guarantee_single_callable
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class StatelessServer:
|
||||||
|
"""
|
||||||
|
Base server class that handles basic concepts like application instance
|
||||||
|
creation/pooling, exception handling, and similar, for stateless protocols
|
||||||
|
(i.e. ones without actual incoming connections to the process)
|
||||||
|
|
||||||
|
Your code should override the handle() method, doing whatever it needs to,
|
||||||
|
and calling get_or_create_application_instance with a unique `scope_id`
|
||||||
|
and `scope` for the scope it wants to get.
|
||||||
|
|
||||||
|
If an application instance is found with the same `scope_id`, you are
|
||||||
|
given its input queue, otherwise one is made for you with the scope provided
|
||||||
|
and you are given that fresh new input queue. Either way, you should do
|
||||||
|
something like:
|
||||||
|
|
||||||
|
input_queue = self.get_or_create_application_instance(
|
||||||
|
"user-123456",
|
||||||
|
{"type": "testprotocol", "user_id": "123456", "username": "andrew"},
|
||||||
|
)
|
||||||
|
input_queue.put_nowait(message)
|
||||||
|
|
||||||
|
If you try and create an application instance and there are already
|
||||||
|
`max_application` instances, the oldest/least recently used one will be
|
||||||
|
reclaimed and shut down to make space.
|
||||||
|
|
||||||
|
Application coroutines that error will be found periodically (every 100ms
|
||||||
|
by default) and have their exceptions printed to the console. Override
|
||||||
|
application_exception() if you want to do more when this happens.
|
||||||
|
|
||||||
|
If you override run(), make sure you handle things like launching the
|
||||||
|
application checker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
application_checker_interval = 0.1
|
||||||
|
|
||||||
|
def __init__(self, application, max_applications=1000):
|
||||||
|
# Parameters
|
||||||
|
self.application = application
|
||||||
|
self.max_applications = max_applications
|
||||||
|
# Initialisation
|
||||||
|
self.application_instances = {}
|
||||||
|
|
||||||
|
### Mainloop and handling
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""
|
||||||
|
Runs the asyncio event loop with our handler loop.
|
||||||
|
"""
|
||||||
|
event_loop = asyncio.get_event_loop()
|
||||||
|
asyncio.ensure_future(self.application_checker())
|
||||||
|
try:
|
||||||
|
event_loop.run_until_complete(self.handle())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logger.info("Exiting due to Ctrl-C/interrupt")
|
||||||
|
|
||||||
|
async def handle(self):
|
||||||
|
raise NotImplementedError("You must implement handle()")
|
||||||
|
|
||||||
|
async def application_send(self, scope, message):
|
||||||
|
"""
|
||||||
|
Receives outbound sends from applications and handles them.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("You must implement application_send()")
|
||||||
|
|
||||||
|
### Application instance management
|
||||||
|
|
||||||
|
def get_or_create_application_instance(self, scope_id, scope):
|
||||||
|
"""
|
||||||
|
Creates an application instance and returns its queue.
|
||||||
|
"""
|
||||||
|
if scope_id in self.application_instances:
|
||||||
|
self.application_instances[scope_id]["last_used"] = time.time()
|
||||||
|
return self.application_instances[scope_id]["input_queue"]
|
||||||
|
# See if we need to delete an old one
|
||||||
|
while len(self.application_instances) > self.max_applications:
|
||||||
|
self.delete_oldest_application_instance()
|
||||||
|
# Make an instance of the application
|
||||||
|
input_queue = asyncio.Queue()
|
||||||
|
application_instance = guarantee_single_callable(self.application)
|
||||||
|
# Run it, and stash the future for later checking
|
||||||
|
future = asyncio.ensure_future(
|
||||||
|
application_instance(
|
||||||
|
scope=scope,
|
||||||
|
receive=input_queue.get,
|
||||||
|
send=lambda message: self.application_send(scope, message),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.application_instances[scope_id] = {
|
||||||
|
"input_queue": input_queue,
|
||||||
|
"future": future,
|
||||||
|
"scope": scope,
|
||||||
|
"last_used": time.time(),
|
||||||
|
}
|
||||||
|
return input_queue
|
||||||
|
|
||||||
|
def delete_oldest_application_instance(self):
|
||||||
|
"""
|
||||||
|
Finds and deletes the oldest application instance
|
||||||
|
"""
|
||||||
|
oldest_time = min(
|
||||||
|
details["last_used"] for details in self.application_instances.values()
|
||||||
|
)
|
||||||
|
for scope_id, details in self.application_instances.items():
|
||||||
|
if details["last_used"] == oldest_time:
|
||||||
|
self.delete_application_instance(scope_id)
|
||||||
|
# Return to make sure we only delete one in case two have
|
||||||
|
# the same oldest time
|
||||||
|
return
|
||||||
|
|
||||||
|
def delete_application_instance(self, scope_id):
|
||||||
|
"""
|
||||||
|
Removes an application instance (makes sure its task is stopped,
|
||||||
|
then removes it from the current set)
|
||||||
|
"""
|
||||||
|
details = self.application_instances[scope_id]
|
||||||
|
del self.application_instances[scope_id]
|
||||||
|
if not details["future"].done():
|
||||||
|
details["future"].cancel()
|
||||||
|
|
||||||
|
async def application_checker(self):
|
||||||
|
"""
|
||||||
|
Goes through the set of current application instance Futures and cleans up
|
||||||
|
any that are done/prints exceptions for any that errored.
|
||||||
|
"""
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(self.application_checker_interval)
|
||||||
|
for scope_id, details in list(self.application_instances.items()):
|
||||||
|
if details["future"].done():
|
||||||
|
exception = details["future"].exception()
|
||||||
|
if exception:
|
||||||
|
await self.application_exception(exception, details)
|
||||||
|
try:
|
||||||
|
del self.application_instances[scope_id]
|
||||||
|
except KeyError:
|
||||||
|
# Exception handling might have already got here before us. That's fine.
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def application_exception(self, exception, application_details):
|
||||||
|
"""
|
||||||
|
Called whenever an application coroutine has an exception.
|
||||||
|
"""
|
||||||
|
logging.error(
|
||||||
|
"Exception inside application: %s\n%s%s",
|
||||||
|
exception,
|
||||||
|
"".join(traceback.format_tb(exception.__traceback__)),
|
||||||
|
f" {exception}",
|
||||||
|
)
|
532
acer-env/lib/python3.10/site-packages/asgiref/sync.py
Normal file
532
acer-env/lib/python3.10/site-packages/asgiref/sync.py
Normal file
@ -0,0 +1,532 @@
|
|||||||
|
import asyncio
|
||||||
|
import asyncio.coroutines
|
||||||
|
import contextvars
|
||||||
|
import functools
|
||||||
|
import inspect
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import warnings
|
||||||
|
import weakref
|
||||||
|
from concurrent.futures import Future, ThreadPoolExecutor
|
||||||
|
from typing import Any, Callable, Dict, Optional, overload
|
||||||
|
|
||||||
|
from .current_thread_executor import CurrentThreadExecutor
|
||||||
|
from .local import Local
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_context(context):
|
||||||
|
# Check for changes in contextvars, and set them to the current
|
||||||
|
# context for downstream consumers
|
||||||
|
for cvar in context:
|
||||||
|
try:
|
||||||
|
if cvar.get() != context.get(cvar):
|
||||||
|
cvar.set(context.get(cvar))
|
||||||
|
except LookupError:
|
||||||
|
cvar.set(context.get(cvar))
|
||||||
|
|
||||||
|
|
||||||
|
def _iscoroutinefunction_or_partial(func: Any) -> bool:
|
||||||
|
# Python < 3.8 does not correctly determine partially wrapped
|
||||||
|
# coroutine functions are coroutine functions, hence the need for
|
||||||
|
# this to exist. Code taken from CPython.
|
||||||
|
if sys.version_info >= (3, 8):
|
||||||
|
return asyncio.iscoroutinefunction(func)
|
||||||
|
else:
|
||||||
|
while inspect.ismethod(func):
|
||||||
|
func = func.__func__
|
||||||
|
while isinstance(func, functools.partial):
|
||||||
|
func = func.func
|
||||||
|
|
||||||
|
return asyncio.iscoroutinefunction(func)
|
||||||
|
|
||||||
|
|
||||||
|
class ThreadSensitiveContext:
|
||||||
|
"""Async context manager to manage context for thread sensitive mode
|
||||||
|
|
||||||
|
This context manager controls which thread pool executor is used when in
|
||||||
|
thread sensitive mode. By default, a single thread pool executor is shared
|
||||||
|
within a process.
|
||||||
|
|
||||||
|
In Python 3.7+, the ThreadSensitiveContext() context manager may be used to
|
||||||
|
specify a thread pool per context.
|
||||||
|
|
||||||
|
This context manager is re-entrant, so only the outer-most call to
|
||||||
|
ThreadSensitiveContext will set the context.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
>>> import time
|
||||||
|
>>> async with ThreadSensitiveContext():
|
||||||
|
... await sync_to_async(time.sleep, 1)()
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.token = None
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
try:
|
||||||
|
SyncToAsync.thread_sensitive_context.get()
|
||||||
|
except LookupError:
|
||||||
|
self.token = SyncToAsync.thread_sensitive_context.set(self)
|
||||||
|
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc, value, tb):
|
||||||
|
if not self.token:
|
||||||
|
return
|
||||||
|
|
||||||
|
executor = SyncToAsync.context_to_thread_executor.pop(self, None)
|
||||||
|
if executor:
|
||||||
|
executor.shutdown()
|
||||||
|
SyncToAsync.thread_sensitive_context.reset(self.token)
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncToSync:
|
||||||
|
"""
|
||||||
|
Utility class which turns an awaitable that only works on the thread with
|
||||||
|
the event loop into a synchronous callable that works in a subthread.
|
||||||
|
|
||||||
|
If the call stack contains an async loop, the code runs there.
|
||||||
|
Otherwise, the code runs in a new loop in a new thread.
|
||||||
|
|
||||||
|
Either way, this thread then pauses and waits to run any thread_sensitive
|
||||||
|
code called from further down the call stack using SyncToAsync, before
|
||||||
|
finally exiting once the async task returns.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Maps launched Tasks to the threads that launched them (for locals impl)
|
||||||
|
launch_map: "Dict[asyncio.Task[object], threading.Thread]" = {}
|
||||||
|
|
||||||
|
# Keeps track of which CurrentThreadExecutor to use. This uses an asgiref
|
||||||
|
# Local, not a threadlocal, so that tasks can work out what their parent used.
|
||||||
|
executors = Local()
|
||||||
|
|
||||||
|
# When we can't find a CurrentThreadExecutor from the context, such as
|
||||||
|
# inside create_task, we'll look it up here from the running event loop.
|
||||||
|
loop_thread_executors: "Dict[asyncio.AbstractEventLoop, CurrentThreadExecutor]" = {}
|
||||||
|
|
||||||
|
def __init__(self, awaitable, force_new_loop=False):
|
||||||
|
if not callable(awaitable) or (
|
||||||
|
not _iscoroutinefunction_or_partial(awaitable)
|
||||||
|
and not _iscoroutinefunction_or_partial(
|
||||||
|
getattr(awaitable, "__call__", awaitable)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
# Python does not have very reliable detection of async functions
|
||||||
|
# (lots of false negatives) so this is just a warning.
|
||||||
|
warnings.warn(
|
||||||
|
"async_to_sync was passed a non-async-marked callable", stacklevel=2
|
||||||
|
)
|
||||||
|
self.awaitable = awaitable
|
||||||
|
try:
|
||||||
|
self.__self__ = self.awaitable.__self__
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
if force_new_loop:
|
||||||
|
# They have asked that we always run in a new sub-loop.
|
||||||
|
self.main_event_loop = None
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
self.main_event_loop = asyncio.get_running_loop()
|
||||||
|
except RuntimeError:
|
||||||
|
# There's no event loop in this thread. Look for the threadlocal if
|
||||||
|
# we're inside SyncToAsync
|
||||||
|
main_event_loop_pid = getattr(
|
||||||
|
SyncToAsync.threadlocal, "main_event_loop_pid", None
|
||||||
|
)
|
||||||
|
# We make sure the parent loop is from the same process - if
|
||||||
|
# they've forked, this is not going to be valid any more (#194)
|
||||||
|
if main_event_loop_pid and main_event_loop_pid == os.getpid():
|
||||||
|
self.main_event_loop = getattr(
|
||||||
|
SyncToAsync.threadlocal, "main_event_loop", None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.main_event_loop = None
|
||||||
|
|
||||||
|
def __call__(self, *args, **kwargs):
|
||||||
|
# You can't call AsyncToSync from a thread with a running event loop
|
||||||
|
try:
|
||||||
|
event_loop = asyncio.get_running_loop()
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if event_loop.is_running():
|
||||||
|
raise RuntimeError(
|
||||||
|
"You cannot use AsyncToSync in the same thread as an async event loop - "
|
||||||
|
"just await the async function directly."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Wrapping context in list so it can be reassigned from within
|
||||||
|
# `main_wrap`.
|
||||||
|
context = [contextvars.copy_context()]
|
||||||
|
|
||||||
|
# Make a future for the return information
|
||||||
|
call_result = Future()
|
||||||
|
# Get the source thread
|
||||||
|
source_thread = threading.current_thread()
|
||||||
|
# Make a CurrentThreadExecutor we'll use to idle in this thread - we
|
||||||
|
# need one for every sync frame, even if there's one above us in the
|
||||||
|
# same thread.
|
||||||
|
if hasattr(self.executors, "current"):
|
||||||
|
old_current_executor = self.executors.current
|
||||||
|
else:
|
||||||
|
old_current_executor = None
|
||||||
|
current_executor = CurrentThreadExecutor()
|
||||||
|
self.executors.current = current_executor
|
||||||
|
loop = None
|
||||||
|
# Use call_soon_threadsafe to schedule a synchronous callback on the
|
||||||
|
# main event loop's thread if it's there, otherwise make a new loop
|
||||||
|
# in this thread.
|
||||||
|
try:
|
||||||
|
awaitable = self.main_wrap(
|
||||||
|
args, kwargs, call_result, source_thread, sys.exc_info(), context
|
||||||
|
)
|
||||||
|
|
||||||
|
if not (self.main_event_loop and self.main_event_loop.is_running()):
|
||||||
|
# Make our own event loop - in a new thread - and run inside that.
|
||||||
|
loop = asyncio.new_event_loop()
|
||||||
|
self.loop_thread_executors[loop] = current_executor
|
||||||
|
loop_executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
loop_future = loop_executor.submit(
|
||||||
|
self._run_event_loop, loop, awaitable
|
||||||
|
)
|
||||||
|
if current_executor:
|
||||||
|
# Run the CurrentThreadExecutor until the future is done
|
||||||
|
current_executor.run_until_future(loop_future)
|
||||||
|
# Wait for future and/or allow for exception propagation
|
||||||
|
loop_future.result()
|
||||||
|
else:
|
||||||
|
# Call it inside the existing loop
|
||||||
|
self.main_event_loop.call_soon_threadsafe(
|
||||||
|
self.main_event_loop.create_task, awaitable
|
||||||
|
)
|
||||||
|
if current_executor:
|
||||||
|
# Run the CurrentThreadExecutor until the future is done
|
||||||
|
current_executor.run_until_future(call_result)
|
||||||
|
finally:
|
||||||
|
# Clean up any executor we were running
|
||||||
|
if loop is not None:
|
||||||
|
del self.loop_thread_executors[loop]
|
||||||
|
if hasattr(self.executors, "current"):
|
||||||
|
del self.executors.current
|
||||||
|
if old_current_executor:
|
||||||
|
self.executors.current = old_current_executor
|
||||||
|
_restore_context(context[0])
|
||||||
|
|
||||||
|
# Wait for results from the future.
|
||||||
|
return call_result.result()
|
||||||
|
|
||||||
|
def _run_event_loop(self, loop, coro):
|
||||||
|
"""
|
||||||
|
Runs the given event loop (designed to be called in a thread).
|
||||||
|
"""
|
||||||
|
asyncio.set_event_loop(loop)
|
||||||
|
try:
|
||||||
|
loop.run_until_complete(coro)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
# mimic asyncio.run() behavior
|
||||||
|
# cancel unexhausted async generators
|
||||||
|
tasks = asyncio.all_tasks(loop)
|
||||||
|
for task in tasks:
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
async def gather():
|
||||||
|
await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
loop.run_until_complete(gather())
|
||||||
|
for task in tasks:
|
||||||
|
if task.cancelled():
|
||||||
|
continue
|
||||||
|
if task.exception() is not None:
|
||||||
|
loop.call_exception_handler(
|
||||||
|
{
|
||||||
|
"message": "unhandled exception during loop shutdown",
|
||||||
|
"exception": task.exception(),
|
||||||
|
"task": task,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if hasattr(loop, "shutdown_asyncgens"):
|
||||||
|
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||||
|
finally:
|
||||||
|
loop.close()
|
||||||
|
asyncio.set_event_loop(self.main_event_loop)
|
||||||
|
|
||||||
|
def __get__(self, parent, objtype):
|
||||||
|
"""
|
||||||
|
Include self for methods
|
||||||
|
"""
|
||||||
|
func = functools.partial(self.__call__, parent)
|
||||||
|
return functools.update_wrapper(func, self.awaitable)
|
||||||
|
|
||||||
|
async def main_wrap(
|
||||||
|
self, args, kwargs, call_result, source_thread, exc_info, context
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Wraps the awaitable with something that puts the result into the
|
||||||
|
result/exception future.
|
||||||
|
"""
|
||||||
|
if context is not None:
|
||||||
|
_restore_context(context[0])
|
||||||
|
|
||||||
|
current_task = SyncToAsync.get_current_task()
|
||||||
|
self.launch_map[current_task] = source_thread
|
||||||
|
try:
|
||||||
|
# If we have an exception, run the function inside the except block
|
||||||
|
# after raising it so exc_info is correctly populated.
|
||||||
|
if exc_info[1]:
|
||||||
|
try:
|
||||||
|
raise exc_info[1]
|
||||||
|
except BaseException:
|
||||||
|
result = await self.awaitable(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
result = await self.awaitable(*args, **kwargs)
|
||||||
|
except BaseException as e:
|
||||||
|
call_result.set_exception(e)
|
||||||
|
else:
|
||||||
|
call_result.set_result(result)
|
||||||
|
finally:
|
||||||
|
del self.launch_map[current_task]
|
||||||
|
|
||||||
|
context[0] = contextvars.copy_context()
|
||||||
|
|
||||||
|
|
||||||
|
class SyncToAsync:
|
||||||
|
"""
|
||||||
|
Utility class which turns a synchronous callable into an awaitable that
|
||||||
|
runs in a threadpool. It also sets a threadlocal inside the thread so
|
||||||
|
calls to AsyncToSync can escape it.
|
||||||
|
|
||||||
|
If thread_sensitive is passed, the code will run in the same thread as any
|
||||||
|
outer code. This is needed for underlying Python code that is not
|
||||||
|
threadsafe (for example, code which handles SQLite database connections).
|
||||||
|
|
||||||
|
If the outermost program is async (i.e. SyncToAsync is outermost), then
|
||||||
|
this will be a dedicated single sub-thread that all sync code runs in,
|
||||||
|
one after the other. If the outermost program is sync (i.e. AsyncToSync is
|
||||||
|
outermost), this will just be the main thread. This is achieved by idling
|
||||||
|
with a CurrentThreadExecutor while AsyncToSync is blocking its sync parent,
|
||||||
|
rather than just blocking.
|
||||||
|
|
||||||
|
If executor is passed in, that will be used instead of the loop's default executor.
|
||||||
|
In order to pass in an executor, thread_sensitive must be set to False, otherwise
|
||||||
|
a TypeError will be raised.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# If they've set ASGI_THREADS, update the default asyncio executor for now
|
||||||
|
if "ASGI_THREADS" in os.environ:
|
||||||
|
# We use get_event_loop here - not get_running_loop - as this will
|
||||||
|
# be run at import time, and we want to update the main thread's loop.
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
loop.set_default_executor(
|
||||||
|
ThreadPoolExecutor(max_workers=int(os.environ["ASGI_THREADS"]))
|
||||||
|
)
|
||||||
|
|
||||||
|
# Maps launched threads to the coroutines that spawned them
|
||||||
|
launch_map: "Dict[threading.Thread, asyncio.Task[object]]" = {}
|
||||||
|
|
||||||
|
# Storage for main event loop references
|
||||||
|
threadlocal = threading.local()
|
||||||
|
|
||||||
|
# Single-thread executor for thread-sensitive code
|
||||||
|
single_thread_executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
|
||||||
|
# Maintain a contextvar for the current execution context. Optionally used
|
||||||
|
# for thread sensitive mode.
|
||||||
|
thread_sensitive_context: "contextvars.ContextVar[str]" = contextvars.ContextVar(
|
||||||
|
"thread_sensitive_context"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Contextvar that is used to detect if the single thread executor
|
||||||
|
# would be awaited on while already being used in the same context
|
||||||
|
deadlock_context: "contextvars.ContextVar[bool]" = contextvars.ContextVar(
|
||||||
|
"deadlock_context"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Maintaining a weak reference to the context ensures that thread pools are
|
||||||
|
# erased once the context goes out of scope. This terminates the thread pool.
|
||||||
|
context_to_thread_executor: "weakref.WeakKeyDictionary[object, ThreadPoolExecutor]" = (
|
||||||
|
weakref.WeakKeyDictionary()
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
func: Callable[..., Any],
|
||||||
|
thread_sensitive: bool = True,
|
||||||
|
executor: Optional["ThreadPoolExecutor"] = None,
|
||||||
|
) -> None:
|
||||||
|
if (
|
||||||
|
not callable(func)
|
||||||
|
or _iscoroutinefunction_or_partial(func)
|
||||||
|
or _iscoroutinefunction_or_partial(getattr(func, "__call__", func))
|
||||||
|
):
|
||||||
|
raise TypeError("sync_to_async can only be applied to sync functions.")
|
||||||
|
self.func = func
|
||||||
|
functools.update_wrapper(self, func)
|
||||||
|
self._thread_sensitive = thread_sensitive
|
||||||
|
self._is_coroutine = asyncio.coroutines._is_coroutine # type: ignore
|
||||||
|
if thread_sensitive and executor is not None:
|
||||||
|
raise TypeError("executor must not be set when thread_sensitive is True")
|
||||||
|
self._executor = executor
|
||||||
|
try:
|
||||||
|
self.__self__ = func.__self__ # type: ignore
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def __call__(self, *args, **kwargs):
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
# Work out what thread to run the code in
|
||||||
|
if self._thread_sensitive:
|
||||||
|
if hasattr(AsyncToSync.executors, "current"):
|
||||||
|
# If we have a parent sync thread above somewhere, use that
|
||||||
|
executor = AsyncToSync.executors.current
|
||||||
|
elif self.thread_sensitive_context and self.thread_sensitive_context.get(
|
||||||
|
None
|
||||||
|
):
|
||||||
|
# If we have a way of retrieving the current context, attempt
|
||||||
|
# to use a per-context thread pool executor
|
||||||
|
thread_sensitive_context = self.thread_sensitive_context.get()
|
||||||
|
|
||||||
|
if thread_sensitive_context in self.context_to_thread_executor:
|
||||||
|
# Re-use thread executor in current context
|
||||||
|
executor = self.context_to_thread_executor[thread_sensitive_context]
|
||||||
|
else:
|
||||||
|
# Create new thread executor in current context
|
||||||
|
executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
self.context_to_thread_executor[thread_sensitive_context] = executor
|
||||||
|
elif loop in AsyncToSync.loop_thread_executors:
|
||||||
|
# Re-use thread executor for running loop
|
||||||
|
executor = AsyncToSync.loop_thread_executors[loop]
|
||||||
|
elif self.deadlock_context and self.deadlock_context.get(False):
|
||||||
|
raise RuntimeError(
|
||||||
|
"Single thread executor already being used, would deadlock"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Otherwise, we run it in a fixed single thread
|
||||||
|
executor = self.single_thread_executor
|
||||||
|
if self.deadlock_context:
|
||||||
|
self.deadlock_context.set(True)
|
||||||
|
else:
|
||||||
|
# Use the passed in executor, or the loop's default if it is None
|
||||||
|
executor = self._executor
|
||||||
|
|
||||||
|
context = contextvars.copy_context()
|
||||||
|
child = functools.partial(self.func, *args, **kwargs)
|
||||||
|
func = context.run
|
||||||
|
args = (child,)
|
||||||
|
kwargs = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Run the code in the right thread
|
||||||
|
future = loop.run_in_executor(
|
||||||
|
executor,
|
||||||
|
functools.partial(
|
||||||
|
self.thread_handler,
|
||||||
|
loop,
|
||||||
|
self.get_current_task(),
|
||||||
|
sys.exc_info(),
|
||||||
|
func,
|
||||||
|
*args,
|
||||||
|
**kwargs,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
ret = await asyncio.wait_for(future, timeout=None)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
_restore_context(context)
|
||||||
|
if self.deadlock_context:
|
||||||
|
self.deadlock_context.set(False)
|
||||||
|
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def __get__(self, parent, objtype):
|
||||||
|
"""
|
||||||
|
Include self for methods
|
||||||
|
"""
|
||||||
|
return functools.partial(self.__call__, parent)
|
||||||
|
|
||||||
|
def thread_handler(self, loop, source_task, exc_info, func, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Wraps the sync application with exception handling.
|
||||||
|
"""
|
||||||
|
# Set the threadlocal for AsyncToSync
|
||||||
|
self.threadlocal.main_event_loop = loop
|
||||||
|
self.threadlocal.main_event_loop_pid = os.getpid()
|
||||||
|
# Set the task mapping (used for the locals module)
|
||||||
|
current_thread = threading.current_thread()
|
||||||
|
if AsyncToSync.launch_map.get(source_task) == current_thread:
|
||||||
|
# Our parent task was launched from this same thread, so don't make
|
||||||
|
# a launch map entry - let it shortcut over us! (and stop infinite loops)
|
||||||
|
parent_set = False
|
||||||
|
else:
|
||||||
|
self.launch_map[current_thread] = source_task
|
||||||
|
parent_set = True
|
||||||
|
# Run the function
|
||||||
|
try:
|
||||||
|
# If we have an exception, run the function inside the except block
|
||||||
|
# after raising it so exc_info is correctly populated.
|
||||||
|
if exc_info[1]:
|
||||||
|
try:
|
||||||
|
raise exc_info[1]
|
||||||
|
except BaseException:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
finally:
|
||||||
|
# Only delete the launch_map parent if we set it, otherwise it is
|
||||||
|
# from someone else.
|
||||||
|
if parent_set:
|
||||||
|
del self.launch_map[current_thread]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_current_task():
|
||||||
|
"""
|
||||||
|
Implementation of asyncio.current_task()
|
||||||
|
that returns None if there is no task.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return asyncio.current_task()
|
||||||
|
except RuntimeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Lowercase aliases (and decorator friendliness)
|
||||||
|
async_to_sync = AsyncToSync
|
||||||
|
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def sync_to_async(
|
||||||
|
func: None = None,
|
||||||
|
thread_sensitive: bool = True,
|
||||||
|
executor: Optional["ThreadPoolExecutor"] = None,
|
||||||
|
) -> Callable[[Callable[..., Any]], SyncToAsync]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def sync_to_async(
|
||||||
|
func: Callable[..., Any],
|
||||||
|
thread_sensitive: bool = True,
|
||||||
|
executor: Optional["ThreadPoolExecutor"] = None,
|
||||||
|
) -> SyncToAsync:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
def sync_to_async(
|
||||||
|
func=None,
|
||||||
|
thread_sensitive=True,
|
||||||
|
executor=None,
|
||||||
|
):
|
||||||
|
if func is None:
|
||||||
|
return lambda f: SyncToAsync(
|
||||||
|
f,
|
||||||
|
thread_sensitive=thread_sensitive,
|
||||||
|
executor=executor,
|
||||||
|
)
|
||||||
|
return SyncToAsync(
|
||||||
|
func,
|
||||||
|
thread_sensitive=thread_sensitive,
|
||||||
|
executor=executor,
|
||||||
|
)
|
97
acer-env/lib/python3.10/site-packages/asgiref/testing.py
Normal file
97
acer-env/lib/python3.10/site-packages/asgiref/testing.py
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
from .compatibility import guarantee_single_callable
|
||||||
|
from .timeout import timeout as async_timeout
|
||||||
|
|
||||||
|
|
||||||
|
class ApplicationCommunicator:
|
||||||
|
"""
|
||||||
|
Runs an ASGI application in a test mode, allowing sending of
|
||||||
|
messages to it and retrieval of messages it sends.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, application, scope):
|
||||||
|
self.application = guarantee_single_callable(application)
|
||||||
|
self.scope = scope
|
||||||
|
self.input_queue = asyncio.Queue()
|
||||||
|
self.output_queue = asyncio.Queue()
|
||||||
|
self.future = asyncio.ensure_future(
|
||||||
|
self.application(scope, self.input_queue.get, self.output_queue.put)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def wait(self, timeout=1):
|
||||||
|
"""
|
||||||
|
Waits for the application to stop itself and returns any exceptions.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
async with async_timeout(timeout):
|
||||||
|
try:
|
||||||
|
await self.future
|
||||||
|
self.future.result()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
if not self.future.done():
|
||||||
|
self.future.cancel()
|
||||||
|
try:
|
||||||
|
await self.future
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def stop(self, exceptions=True):
|
||||||
|
if not self.future.done():
|
||||||
|
self.future.cancel()
|
||||||
|
elif exceptions:
|
||||||
|
# Give a chance to raise any exceptions
|
||||||
|
self.future.result()
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
# Clean up on deletion
|
||||||
|
try:
|
||||||
|
self.stop(exceptions=False)
|
||||||
|
except RuntimeError:
|
||||||
|
# Event loop already stopped
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def send_input(self, message):
|
||||||
|
"""
|
||||||
|
Sends a single message to the application
|
||||||
|
"""
|
||||||
|
# Give it the message
|
||||||
|
await self.input_queue.put(message)
|
||||||
|
|
||||||
|
async def receive_output(self, timeout=1):
|
||||||
|
"""
|
||||||
|
Receives a single message from the application, with optional timeout.
|
||||||
|
"""
|
||||||
|
# Make sure there's not an exception to raise from the task
|
||||||
|
if self.future.done():
|
||||||
|
self.future.result()
|
||||||
|
# Wait and receive the message
|
||||||
|
try:
|
||||||
|
async with async_timeout(timeout):
|
||||||
|
return await self.output_queue.get()
|
||||||
|
except asyncio.TimeoutError as e:
|
||||||
|
# See if we have another error to raise inside
|
||||||
|
if self.future.done():
|
||||||
|
self.future.result()
|
||||||
|
else:
|
||||||
|
self.future.cancel()
|
||||||
|
try:
|
||||||
|
await self.future
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
raise e
|
||||||
|
|
||||||
|
async def receive_nothing(self, timeout=0.1, interval=0.01):
|
||||||
|
"""
|
||||||
|
Checks that there is no message to receive in the given time.
|
||||||
|
"""
|
||||||
|
# `interval` has precedence over `timeout`
|
||||||
|
start = time.monotonic()
|
||||||
|
while time.monotonic() - start < timeout:
|
||||||
|
if not self.output_queue.empty():
|
||||||
|
return False
|
||||||
|
await asyncio.sleep(interval)
|
||||||
|
return self.output_queue.empty()
|
112
acer-env/lib/python3.10/site-packages/asgiref/timeout.py
Normal file
112
acer-env/lib/python3.10/site-packages/asgiref/timeout.py
Normal file
@ -0,0 +1,112 @@
|
|||||||
|
# This code is originally sourced from the aio-libs project "async_timeout",
|
||||||
|
# under the Apache 2.0 license. You may see the original project at
|
||||||
|
# https://github.com/aio-libs/async-timeout
|
||||||
|
|
||||||
|
# It is vendored here to reduce chain-dependencies on this library, and
|
||||||
|
# modified slightly to remove some features we don't use.
|
||||||
|
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from types import TracebackType
|
||||||
|
from typing import Any, Optional, Type
|
||||||
|
|
||||||
|
|
||||||
|
class timeout:
|
||||||
|
"""timeout context manager.
|
||||||
|
|
||||||
|
Useful in cases when you want to apply timeout logic around block
|
||||||
|
of code or in cases when asyncio.wait_for is not suitable. For example:
|
||||||
|
|
||||||
|
>>> with timeout(0.001):
|
||||||
|
... async with aiohttp.get('https://github.com') as r:
|
||||||
|
... await r.text()
|
||||||
|
|
||||||
|
|
||||||
|
timeout - value in seconds or None to disable timeout logic
|
||||||
|
loop - asyncio compatible event loop
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
timeout: Optional[float],
|
||||||
|
*,
|
||||||
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
||||||
|
) -> None:
|
||||||
|
self._timeout = timeout
|
||||||
|
if loop is None:
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
self._loop = loop
|
||||||
|
self._task = None # type: Optional[asyncio.Task[Any]]
|
||||||
|
self._cancelled = False
|
||||||
|
self._cancel_handler = None # type: Optional[asyncio.Handle]
|
||||||
|
self._cancel_at = None # type: Optional[float]
|
||||||
|
|
||||||
|
def __enter__(self) -> "timeout":
|
||||||
|
return self._do_enter()
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self,
|
||||||
|
exc_type: Type[BaseException],
|
||||||
|
exc_val: BaseException,
|
||||||
|
exc_tb: TracebackType,
|
||||||
|
) -> Optional[bool]:
|
||||||
|
self._do_exit(exc_type)
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "timeout":
|
||||||
|
return self._do_enter()
|
||||||
|
|
||||||
|
async def __aexit__(
|
||||||
|
self,
|
||||||
|
exc_type: Type[BaseException],
|
||||||
|
exc_val: BaseException,
|
||||||
|
exc_tb: TracebackType,
|
||||||
|
) -> None:
|
||||||
|
self._do_exit(exc_type)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def expired(self) -> bool:
|
||||||
|
return self._cancelled
|
||||||
|
|
||||||
|
@property
|
||||||
|
def remaining(self) -> Optional[float]:
|
||||||
|
if self._cancel_at is not None:
|
||||||
|
return max(self._cancel_at - self._loop.time(), 0.0)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _do_enter(self) -> "timeout":
|
||||||
|
# Support Tornado 5- without timeout
|
||||||
|
# Details: https://github.com/python/asyncio/issues/392
|
||||||
|
if self._timeout is None:
|
||||||
|
return self
|
||||||
|
|
||||||
|
self._task = asyncio.current_task(self._loop)
|
||||||
|
if self._task is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Timeout context manager should be used " "inside a task"
|
||||||
|
)
|
||||||
|
|
||||||
|
if self._timeout <= 0:
|
||||||
|
self._loop.call_soon(self._cancel_task)
|
||||||
|
return self
|
||||||
|
|
||||||
|
self._cancel_at = self._loop.time() + self._timeout
|
||||||
|
self._cancel_handler = self._loop.call_at(self._cancel_at, self._cancel_task)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def _do_exit(self, exc_type: Type[BaseException]) -> None:
|
||||||
|
if exc_type is asyncio.CancelledError and self._cancelled:
|
||||||
|
self._cancel_handler = None
|
||||||
|
self._task = None
|
||||||
|
raise asyncio.TimeoutError
|
||||||
|
if self._timeout is not None and self._cancel_handler is not None:
|
||||||
|
self._cancel_handler.cancel()
|
||||||
|
self._cancel_handler = None
|
||||||
|
self._task = None
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _cancel_task(self) -> None:
|
||||||
|
if self._task is not None:
|
||||||
|
self._task.cancel()
|
||||||
|
self._cancelled = True
|
242
acer-env/lib/python3.10/site-packages/asgiref/typing.py
Normal file
242
acer-env/lib/python3.10/site-packages/asgiref/typing.py
Normal file
@ -0,0 +1,242 @@
|
|||||||
|
import sys
|
||||||
|
from typing import Awaitable, Callable, Dict, Iterable, Optional, Tuple, Type, Union
|
||||||
|
|
||||||
|
if sys.version_info >= (3, 8):
|
||||||
|
from typing import Literal, Protocol, TypedDict
|
||||||
|
else:
|
||||||
|
from typing_extensions import Literal, Protocol, TypedDict
|
||||||
|
|
||||||
|
__all__ = (
|
||||||
|
"ASGIVersions",
|
||||||
|
"HTTPScope",
|
||||||
|
"WebSocketScope",
|
||||||
|
"LifespanScope",
|
||||||
|
"WWWScope",
|
||||||
|
"Scope",
|
||||||
|
"HTTPRequestEvent",
|
||||||
|
"HTTPResponseStartEvent",
|
||||||
|
"HTTPResponseBodyEvent",
|
||||||
|
"HTTPServerPushEvent",
|
||||||
|
"HTTPDisconnectEvent",
|
||||||
|
"WebSocketConnectEvent",
|
||||||
|
"WebSocketAcceptEvent",
|
||||||
|
"WebSocketReceiveEvent",
|
||||||
|
"WebSocketSendEvent",
|
||||||
|
"WebSocketResponseStartEvent",
|
||||||
|
"WebSocketResponseBodyEvent",
|
||||||
|
"WebSocketDisconnectEvent",
|
||||||
|
"WebSocketCloseEvent",
|
||||||
|
"LifespanStartupEvent",
|
||||||
|
"LifespanShutdownEvent",
|
||||||
|
"LifespanStartupCompleteEvent",
|
||||||
|
"LifespanStartupFailedEvent",
|
||||||
|
"LifespanShutdownCompleteEvent",
|
||||||
|
"LifespanShutdownFailedEvent",
|
||||||
|
"ASGIReceiveEvent",
|
||||||
|
"ASGISendEvent",
|
||||||
|
"ASGIReceiveCallable",
|
||||||
|
"ASGISendCallable",
|
||||||
|
"ASGI2Protocol",
|
||||||
|
"ASGI2Application",
|
||||||
|
"ASGI3Application",
|
||||||
|
"ASGIApplication",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ASGIVersions(TypedDict):
|
||||||
|
spec_version: str
|
||||||
|
version: Union[Literal["2.0"], Literal["3.0"]]
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPScope(TypedDict):
|
||||||
|
type: Literal["http"]
|
||||||
|
asgi: ASGIVersions
|
||||||
|
http_version: str
|
||||||
|
method: str
|
||||||
|
scheme: str
|
||||||
|
path: str
|
||||||
|
raw_path: bytes
|
||||||
|
query_string: bytes
|
||||||
|
root_path: str
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
client: Optional[Tuple[str, int]]
|
||||||
|
server: Optional[Tuple[str, Optional[int]]]
|
||||||
|
extensions: Optional[Dict[str, Dict[object, object]]]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketScope(TypedDict):
|
||||||
|
type: Literal["websocket"]
|
||||||
|
asgi: ASGIVersions
|
||||||
|
http_version: str
|
||||||
|
scheme: str
|
||||||
|
path: str
|
||||||
|
raw_path: bytes
|
||||||
|
query_string: bytes
|
||||||
|
root_path: str
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
client: Optional[Tuple[str, int]]
|
||||||
|
server: Optional[Tuple[str, Optional[int]]]
|
||||||
|
subprotocols: Iterable[str]
|
||||||
|
extensions: Optional[Dict[str, Dict[object, object]]]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanScope(TypedDict):
|
||||||
|
type: Literal["lifespan"]
|
||||||
|
asgi: ASGIVersions
|
||||||
|
|
||||||
|
|
||||||
|
WWWScope = Union[HTTPScope, WebSocketScope]
|
||||||
|
Scope = Union[HTTPScope, WebSocketScope, LifespanScope]
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPRequestEvent(TypedDict):
|
||||||
|
type: Literal["http.request"]
|
||||||
|
body: bytes
|
||||||
|
more_body: bool
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPResponseStartEvent(TypedDict):
|
||||||
|
type: Literal["http.response.start"]
|
||||||
|
status: int
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPResponseBodyEvent(TypedDict):
|
||||||
|
type: Literal["http.response.body"]
|
||||||
|
body: bytes
|
||||||
|
more_body: bool
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPServerPushEvent(TypedDict):
|
||||||
|
type: Literal["http.response.push"]
|
||||||
|
path: str
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPDisconnectEvent(TypedDict):
|
||||||
|
type: Literal["http.disconnect"]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketConnectEvent(TypedDict):
|
||||||
|
type: Literal["websocket.connect"]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketAcceptEvent(TypedDict):
|
||||||
|
type: Literal["websocket.accept"]
|
||||||
|
subprotocol: Optional[str]
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketReceiveEvent(TypedDict):
|
||||||
|
type: Literal["websocket.receive"]
|
||||||
|
bytes: Optional[bytes]
|
||||||
|
text: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketSendEvent(TypedDict):
|
||||||
|
type: Literal["websocket.send"]
|
||||||
|
bytes: Optional[bytes]
|
||||||
|
text: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketResponseStartEvent(TypedDict):
|
||||||
|
type: Literal["websocket.http.response.start"]
|
||||||
|
status: int
|
||||||
|
headers: Iterable[Tuple[bytes, bytes]]
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketResponseBodyEvent(TypedDict):
|
||||||
|
type: Literal["websocket.http.response.body"]
|
||||||
|
body: bytes
|
||||||
|
more_body: bool
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketDisconnectEvent(TypedDict):
|
||||||
|
type: Literal["websocket.disconnect"]
|
||||||
|
code: int
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketCloseEvent(TypedDict):
|
||||||
|
type: Literal["websocket.close"]
|
||||||
|
code: int
|
||||||
|
reason: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanStartupEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.startup"]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanShutdownEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.shutdown"]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanStartupCompleteEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.startup.complete"]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanStartupFailedEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.startup.failed"]
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanShutdownCompleteEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.shutdown.complete"]
|
||||||
|
|
||||||
|
|
||||||
|
class LifespanShutdownFailedEvent(TypedDict):
|
||||||
|
type: Literal["lifespan.shutdown.failed"]
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
ASGIReceiveEvent = Union[
|
||||||
|
HTTPRequestEvent,
|
||||||
|
HTTPDisconnectEvent,
|
||||||
|
WebSocketConnectEvent,
|
||||||
|
WebSocketReceiveEvent,
|
||||||
|
WebSocketDisconnectEvent,
|
||||||
|
LifespanStartupEvent,
|
||||||
|
LifespanShutdownEvent,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
ASGISendEvent = Union[
|
||||||
|
HTTPResponseStartEvent,
|
||||||
|
HTTPResponseBodyEvent,
|
||||||
|
HTTPServerPushEvent,
|
||||||
|
HTTPDisconnectEvent,
|
||||||
|
WebSocketAcceptEvent,
|
||||||
|
WebSocketSendEvent,
|
||||||
|
WebSocketResponseStartEvent,
|
||||||
|
WebSocketResponseBodyEvent,
|
||||||
|
WebSocketCloseEvent,
|
||||||
|
LifespanStartupCompleteEvent,
|
||||||
|
LifespanStartupFailedEvent,
|
||||||
|
LifespanShutdownCompleteEvent,
|
||||||
|
LifespanShutdownFailedEvent,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
ASGIReceiveCallable = Callable[[], Awaitable[ASGIReceiveEvent]]
|
||||||
|
ASGISendCallable = Callable[[ASGISendEvent], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
|
class ASGI2Protocol(Protocol):
|
||||||
|
def __init__(self, scope: Scope) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
async def __call__(
|
||||||
|
self, receive: ASGIReceiveCallable, send: ASGISendCallable
|
||||||
|
) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
ASGI2Application = Type[ASGI2Protocol]
|
||||||
|
ASGI3Application = Callable[
|
||||||
|
[
|
||||||
|
Scope,
|
||||||
|
ASGIReceiveCallable,
|
||||||
|
ASGISendCallable,
|
||||||
|
],
|
||||||
|
Awaitable[None],
|
||||||
|
]
|
||||||
|
ASGIApplication = Union[ASGI2Application, ASGI3Application]
|
162
acer-env/lib/python3.10/site-packages/asgiref/wsgi.py
Normal file
162
acer-env/lib/python3.10/site-packages/asgiref/wsgi.py
Normal file
@ -0,0 +1,162 @@
|
|||||||
|
from io import BytesIO
|
||||||
|
from tempfile import SpooledTemporaryFile
|
||||||
|
|
||||||
|
from asgiref.sync import AsyncToSync, sync_to_async
|
||||||
|
|
||||||
|
|
||||||
|
class WsgiToAsgi:
|
||||||
|
"""
|
||||||
|
Wraps a WSGI application to make it into an ASGI application.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, wsgi_application):
|
||||||
|
self.wsgi_application = wsgi_application
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
"""
|
||||||
|
ASGI application instantiation point.
|
||||||
|
We return a new WsgiToAsgiInstance here with the WSGI app
|
||||||
|
and the scope, ready to respond when it is __call__ed.
|
||||||
|
"""
|
||||||
|
await WsgiToAsgiInstance(self.wsgi_application)(scope, receive, send)
|
||||||
|
|
||||||
|
|
||||||
|
class WsgiToAsgiInstance:
|
||||||
|
"""
|
||||||
|
Per-socket instance of a wrapped WSGI application
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, wsgi_application):
|
||||||
|
self.wsgi_application = wsgi_application
|
||||||
|
self.response_started = False
|
||||||
|
self.response_content_length = None
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if scope["type"] != "http":
|
||||||
|
raise ValueError("WSGI wrapper received a non-HTTP scope")
|
||||||
|
self.scope = scope
|
||||||
|
with SpooledTemporaryFile(max_size=65536) as body:
|
||||||
|
# Alright, wait for the http.request messages
|
||||||
|
while True:
|
||||||
|
message = await receive()
|
||||||
|
if message["type"] != "http.request":
|
||||||
|
raise ValueError("WSGI wrapper received a non-HTTP-request message")
|
||||||
|
body.write(message.get("body", b""))
|
||||||
|
if not message.get("more_body"):
|
||||||
|
break
|
||||||
|
body.seek(0)
|
||||||
|
# Wrap send so it can be called from the subthread
|
||||||
|
self.sync_send = AsyncToSync(send)
|
||||||
|
# Call the WSGI app
|
||||||
|
await self.run_wsgi_app(body)
|
||||||
|
|
||||||
|
def build_environ(self, scope, body):
|
||||||
|
"""
|
||||||
|
Builds a scope and request body into a WSGI environ object.
|
||||||
|
"""
|
||||||
|
environ = {
|
||||||
|
"REQUEST_METHOD": scope["method"],
|
||||||
|
"SCRIPT_NAME": scope.get("root_path", "").encode("utf8").decode("latin1"),
|
||||||
|
"PATH_INFO": scope["path"].encode("utf8").decode("latin1"),
|
||||||
|
"QUERY_STRING": scope["query_string"].decode("ascii"),
|
||||||
|
"SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"],
|
||||||
|
"wsgi.version": (1, 0),
|
||||||
|
"wsgi.url_scheme": scope.get("scheme", "http"),
|
||||||
|
"wsgi.input": body,
|
||||||
|
"wsgi.errors": BytesIO(),
|
||||||
|
"wsgi.multithread": True,
|
||||||
|
"wsgi.multiprocess": True,
|
||||||
|
"wsgi.run_once": False,
|
||||||
|
}
|
||||||
|
# Get server name and port - required in WSGI, not in ASGI
|
||||||
|
if "server" in scope:
|
||||||
|
environ["SERVER_NAME"] = scope["server"][0]
|
||||||
|
environ["SERVER_PORT"] = str(scope["server"][1])
|
||||||
|
else:
|
||||||
|
environ["SERVER_NAME"] = "localhost"
|
||||||
|
environ["SERVER_PORT"] = "80"
|
||||||
|
|
||||||
|
if "client" in scope:
|
||||||
|
environ["REMOTE_ADDR"] = scope["client"][0]
|
||||||
|
|
||||||
|
# Go through headers and make them into environ entries
|
||||||
|
for name, value in self.scope.get("headers", []):
|
||||||
|
name = name.decode("latin1")
|
||||||
|
if name == "content-length":
|
||||||
|
corrected_name = "CONTENT_LENGTH"
|
||||||
|
elif name == "content-type":
|
||||||
|
corrected_name = "CONTENT_TYPE"
|
||||||
|
else:
|
||||||
|
corrected_name = "HTTP_%s" % name.upper().replace("-", "_")
|
||||||
|
# HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case
|
||||||
|
value = value.decode("latin1")
|
||||||
|
if corrected_name in environ:
|
||||||
|
value = environ[corrected_name] + "," + value
|
||||||
|
environ[corrected_name] = value
|
||||||
|
return environ
|
||||||
|
|
||||||
|
def start_response(self, status, response_headers, exc_info=None):
|
||||||
|
"""
|
||||||
|
WSGI start_response callable.
|
||||||
|
"""
|
||||||
|
# Don't allow re-calling once response has begun
|
||||||
|
if self.response_started:
|
||||||
|
raise exc_info[1].with_traceback(exc_info[2])
|
||||||
|
# Don't allow re-calling without exc_info
|
||||||
|
if hasattr(self, "response_start") and exc_info is None:
|
||||||
|
raise ValueError(
|
||||||
|
"You cannot call start_response a second time without exc_info"
|
||||||
|
)
|
||||||
|
# Extract status code
|
||||||
|
status_code, _ = status.split(" ", 1)
|
||||||
|
status_code = int(status_code)
|
||||||
|
# Extract headers
|
||||||
|
headers = [
|
||||||
|
(name.lower().encode("ascii"), value.encode("ascii"))
|
||||||
|
for name, value in response_headers
|
||||||
|
]
|
||||||
|
# Extract content-length
|
||||||
|
self.response_content_length = None
|
||||||
|
for name, value in response_headers:
|
||||||
|
if name.lower() == "content-length":
|
||||||
|
self.response_content_length = int(value)
|
||||||
|
# Build and send response start message.
|
||||||
|
self.response_start = {
|
||||||
|
"type": "http.response.start",
|
||||||
|
"status": status_code,
|
||||||
|
"headers": headers,
|
||||||
|
}
|
||||||
|
|
||||||
|
@sync_to_async
|
||||||
|
def run_wsgi_app(self, body):
|
||||||
|
"""
|
||||||
|
Called in a subthread to run the WSGI app. We encapsulate like
|
||||||
|
this so that the start_response callable is called in the same thread.
|
||||||
|
"""
|
||||||
|
# Translate the scope and incoming request body into a WSGI environ
|
||||||
|
environ = self.build_environ(self.scope, body)
|
||||||
|
# Run the WSGI app
|
||||||
|
bytes_sent = 0
|
||||||
|
for output in self.wsgi_application(environ, self.start_response):
|
||||||
|
# If this is the first response, include the response headers
|
||||||
|
if not self.response_started:
|
||||||
|
self.response_started = True
|
||||||
|
self.sync_send(self.response_start)
|
||||||
|
# If the application supplies a Content-Length header
|
||||||
|
if self.response_content_length is not None:
|
||||||
|
# The server should not transmit more bytes to the client than the header allows
|
||||||
|
bytes_allowed = self.response_content_length - bytes_sent
|
||||||
|
if len(output) > bytes_allowed:
|
||||||
|
output = output[:bytes_allowed]
|
||||||
|
self.sync_send(
|
||||||
|
{"type": "http.response.body", "body": output, "more_body": True}
|
||||||
|
)
|
||||||
|
bytes_sent += len(output)
|
||||||
|
# The server should stop iterating over the response when enough data has been sent
|
||||||
|
if bytes_sent == self.response_content_length:
|
||||||
|
break
|
||||||
|
# Close connection
|
||||||
|
if not self.response_started:
|
||||||
|
self.response_started = True
|
||||||
|
self.sync_send(self.response_start)
|
||||||
|
self.sync_send({"type": "http.response.body"})
|
@ -0,0 +1 @@
|
|||||||
|
pip
|
@ -0,0 +1,119 @@
|
|||||||
|
Metadata-Version: 2.1
|
||||||
|
Name: beautifulsoup4
|
||||||
|
Version: 4.11.1
|
||||||
|
Summary: Screen-scraping library
|
||||||
|
Home-page: https://www.crummy.com/software/BeautifulSoup/bs4/
|
||||||
|
Author: Leonard Richardson
|
||||||
|
Author-email: leonardr@segfault.org
|
||||||
|
License: MIT
|
||||||
|
Download-URL: https://www.crummy.com/software/BeautifulSoup/bs4/download/
|
||||||
|
Platform: UNKNOWN
|
||||||
|
Classifier: Development Status :: 5 - Production/Stable
|
||||||
|
Classifier: Intended Audience :: Developers
|
||||||
|
Classifier: License :: OSI Approved :: MIT License
|
||||||
|
Classifier: Programming Language :: Python
|
||||||
|
Classifier: Programming Language :: Python :: 3
|
||||||
|
Classifier: Topic :: Text Processing :: Markup :: HTML
|
||||||
|
Classifier: Topic :: Text Processing :: Markup :: XML
|
||||||
|
Classifier: Topic :: Text Processing :: Markup :: SGML
|
||||||
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||||
|
Requires-Python: >=3.6.0
|
||||||
|
Description-Content-Type: text/markdown
|
||||||
|
Provides-Extra: lxml
|
||||||
|
Provides-Extra: html5lib
|
||||||
|
Requires-Dist: soupsieve (>1.2)
|
||||||
|
Provides-Extra: html5lib
|
||||||
|
Requires-Dist: html5lib; extra == 'html5lib'
|
||||||
|
Provides-Extra: lxml
|
||||||
|
Requires-Dist: lxml; extra == 'lxml'
|
||||||
|
|
||||||
|
Beautiful Soup is a library that makes it easy to scrape information
|
||||||
|
from web pages. It sits atop an HTML or XML parser, providing Pythonic
|
||||||
|
idioms for iterating, searching, and modifying the parse tree.
|
||||||
|
|
||||||
|
# Quick start
|
||||||
|
|
||||||
|
```
|
||||||
|
>>> from bs4 import BeautifulSoup
|
||||||
|
>>> soup = BeautifulSoup("<p>Some<b>bad<i>HTML")
|
||||||
|
>>> print(soup.prettify())
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<p>
|
||||||
|
Some
|
||||||
|
<b>
|
||||||
|
bad
|
||||||
|
<i>
|
||||||
|
HTML
|
||||||
|
</i>
|
||||||
|
</b>
|
||||||
|
</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
>>> soup.find(text="bad")
|
||||||
|
'bad'
|
||||||
|
>>> soup.i
|
||||||
|
<i>HTML</i>
|
||||||
|
#
|
||||||
|
>>> soup = BeautifulSoup("<tag1>Some<tag2/>bad<tag3>XML", "xml")
|
||||||
|
#
|
||||||
|
>>> print(soup.prettify())
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<tag1>
|
||||||
|
Some
|
||||||
|
<tag2/>
|
||||||
|
bad
|
||||||
|
<tag3>
|
||||||
|
XML
|
||||||
|
</tag3>
|
||||||
|
</tag1>
|
||||||
|
```
|
||||||
|
|
||||||
|
To go beyond the basics, [comprehensive documentation is available](https://www.crummy.com/software/BeautifulSoup/bs4/doc/).
|
||||||
|
|
||||||
|
# Links
|
||||||
|
|
||||||
|
* [Homepage](https://www.crummy.com/software/BeautifulSoup/bs4/)
|
||||||
|
* [Documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)
|
||||||
|
* [Discussion group](https://groups.google.com/group/beautifulsoup/)
|
||||||
|
* [Development](https://code.launchpad.net/beautifulsoup/)
|
||||||
|
* [Bug tracker](https://bugs.launchpad.net/beautifulsoup/)
|
||||||
|
* [Complete changelog](https://bazaar.launchpad.net/~leonardr/beautifulsoup/bs4/view/head:/CHANGELOG)
|
||||||
|
|
||||||
|
# Note on Python 2 sunsetting
|
||||||
|
|
||||||
|
Beautiful Soup's support for Python 2 was discontinued on December 31,
|
||||||
|
2020: one year after the sunset date for Python 2 itself. From this
|
||||||
|
point onward, new Beautiful Soup development will exclusively target
|
||||||
|
Python 3. The final release of Beautiful Soup 4 to support Python 2
|
||||||
|
was 4.9.3.
|
||||||
|
|
||||||
|
# Supporting the project
|
||||||
|
|
||||||
|
If you use Beautiful Soup as part of your professional work, please consider a
|
||||||
|
[Tidelift subscription](https://tidelift.com/subscription/pkg/pypi-beautifulsoup4?utm_source=pypi-beautifulsoup4&utm_medium=referral&utm_campaign=readme).
|
||||||
|
This will support many of the free software projects your organization
|
||||||
|
depends on, not just Beautiful Soup.
|
||||||
|
|
||||||
|
If you use Beautiful Soup for personal projects, the best way to say
|
||||||
|
thank you is to read
|
||||||
|
[Tool Safety](https://www.crummy.com/software/BeautifulSoup/zine/), a zine I
|
||||||
|
wrote about what Beautiful Soup has taught me about software
|
||||||
|
development.
|
||||||
|
|
||||||
|
# Building the documentation
|
||||||
|
|
||||||
|
The bs4/doc/ directory contains full documentation in Sphinx
|
||||||
|
format. Run `make html` in that directory to create HTML
|
||||||
|
documentation.
|
||||||
|
|
||||||
|
# Running the unit tests
|
||||||
|
|
||||||
|
Beautiful Soup supports unit test discovery using Pytest:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ pytest
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,54 @@
|
|||||||
|
beautifulsoup4-4.11.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||||
|
beautifulsoup4-4.11.1.dist-info/METADATA,sha256=zefuevXradYVap7gKfvpSwLzvs8FSBONKfY8j5HIj-k,3525
|
||||||
|
beautifulsoup4-4.11.1.dist-info/RECORD,,
|
||||||
|
beautifulsoup4-4.11.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||||
|
beautifulsoup4-4.11.1.dist-info/WHEEL,sha256=NzFAKnL7g-U64xnS1s5e3mJnxKpOTeOtlXdFwS9yNXI,92
|
||||||
|
beautifulsoup4-4.11.1.dist-info/top_level.txt,sha256=gpUVJcTwW3q7-QGp6tAEomZsskknmgSqVe6xn1C0jJI,26
|
||||||
|
bs4/__init__.py,sha256=v5VuQqegAzN3bo4kjs6fT0391zvKQGhvBlV841aJ86A,32832
|
||||||
|
bs4/__pycache__/__init__.cpython-310.pyc,,
|
||||||
|
bs4/__pycache__/dammit.cpython-310.pyc,,
|
||||||
|
bs4/__pycache__/diagnose.cpython-310.pyc,,
|
||||||
|
bs4/__pycache__/element.cpython-310.pyc,,
|
||||||
|
bs4/__pycache__/formatter.cpython-310.pyc,,
|
||||||
|
bs4/builder/__init__.py,sha256=Ny7NmZu3XDndp_elCrz2W8K9OUjrAjHQuYianJR6RDM,24378
|
||||||
|
bs4/builder/__pycache__/__init__.cpython-310.pyc,,
|
||||||
|
bs4/builder/__pycache__/_html5lib.cpython-310.pyc,,
|
||||||
|
bs4/builder/__pycache__/_htmlparser.cpython-310.pyc,,
|
||||||
|
bs4/builder/__pycache__/_lxml.cpython-310.pyc,,
|
||||||
|
bs4/builder/_html5lib.py,sha256=YFOKZf3E22AEp_V2k49U5LmF26bpSZixZrZGSwM_iCU,18946
|
||||||
|
bs4/builder/_htmlparser.py,sha256=6RYsAXWb_ppMZGB-7lpEmJQ73tpzSu_CJ5QPaFcJu-Y,19169
|
||||||
|
bs4/builder/_lxml.py,sha256=ik6BFGnxAzV2-21S_Wc-7ZeA174muSA_ZhmpnAe3g0E,14904
|
||||||
|
bs4/dammit.py,sha256=G0cQfsEqfwJ-FIQMkXgCJwSHMn7t9vPepCrud6fZEKk,41158
|
||||||
|
bs4/diagnose.py,sha256=MRbN2bJSpa8VFt8HemqP8BK9hL5ronCxZmrfGRZYwBg,7911
|
||||||
|
bs4/element.py,sha256=jjP-cIA3oWJrAhky61wayot92SQFKzJMEE8My6uTPDM,86753
|
||||||
|
bs4/formatter.py,sha256=f5UBtvW9twrrQeLjBeLaKp4ntZpUDmUBY8jF3BiLraM,7206
|
||||||
|
bs4/tests/__init__.py,sha256=yFvfhDv5vOeho6NEZcI0AIpmuEe9bvC7TsqNp3hDGxQ,49185
|
||||||
|
bs4/tests/__pycache__/__init__.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_builder.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_builder_registry.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_dammit.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_docs.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_element.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_formatter.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_html5lib.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_htmlparser.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_lxml.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_navigablestring.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_pageelement.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_soup.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_tag.cpython-310.pyc,,
|
||||||
|
bs4/tests/__pycache__/test_tree.cpython-310.pyc,,
|
||||||
|
bs4/tests/test_builder.py,sha256=nc2JE5EMrEf-p24qhf2R8qAV5PpFiOuNpYCmtmCjlTI,1115
|
||||||
|
bs4/tests/test_builder_registry.py,sha256=_Vh2CyYzv4BKoJdCa7s4lsBLUQptskfgiSn3U3810CQ,5068
|
||||||
|
bs4/tests/test_dammit.py,sha256=lS3EWCYCtxVE4fC_J2eTcXiChhuaL4Vcbo94B5geIL4,15680
|
||||||
|
bs4/tests/test_docs.py,sha256=xoAxnUfoQ7aRqGImwW_9BJDU8WNMZHIuvWqVepvWXt8,1127
|
||||||
|
bs4/tests/test_element.py,sha256=92oRSRoGk8gIXAbAGHErKzocx2MK32TqcQdUJ-dGQMo,2377
|
||||||
|
bs4/tests/test_formatter.py,sha256=0qV9H7mMDBcnFFH-dwNCrSm2zNi_40WMB2GMcV35PoY,4128
|
||||||
|
bs4/tests/test_html5lib.py,sha256=X6r13jfJ-OmG6SL_hyfFNXWs7sEEq_1TmCzCJclxvbA,8246
|
||||||
|
bs4/tests/test_htmlparser.py,sha256=BFCspIdhkr8Bss-kHufeNcwa_lvJpVWKgJskPoZgZ7E,5532
|
||||||
|
bs4/tests/test_lxml.py,sha256=deaf1YOrR8I0T5yZAV4TDxcAXHzVhdlnsSajGpBoxs0,7376
|
||||||
|
bs4/tests/test_navigablestring.py,sha256=RGSgziNf7cZnYdEPsoqL1B2I68TUJp1JmEQVxbh_ryA,5081
|
||||||
|
bs4/tests/test_pageelement.py,sha256=fpOU3W5IAz92b0A2VxWKkI5pApObMB17cNzXNF85FfA,27792
|
||||||
|
bs4/tests/test_soup.py,sha256=CUnK-rDccIlKMLBP4AweCqRDbPt3Lqzln_BpnBgKm4M,17810
|
||||||
|
bs4/tests/test_tag.py,sha256=f19uie7QehvgvhIqNWfjDRR4TKa-ftm_RRoo6LXZyqk,9016
|
||||||
|
bs4/tests/test_tree.py,sha256=y9Qvs8nnYj6RnGRSxtoYQQwSja-DlbtukCVs0neVwyU,47557
|
@ -0,0 +1,5 @@
|
|||||||
|
Wheel-Version: 1.0
|
||||||
|
Generator: bdist_wheel (0.31.1)
|
||||||
|
Root-Is-Purelib: true
|
||||||
|
Tag: py3-none-any
|
||||||
|
|
@ -0,0 +1,3 @@
|
|||||||
|
bs4
|
||||||
|
bs4/builder
|
||||||
|
bs4/tests
|
812
acer-env/lib/python3.10/site-packages/bs4/__init__.py
Normal file
812
acer-env/lib/python3.10/site-packages/bs4/__init__.py
Normal file
@ -0,0 +1,812 @@
|
|||||||
|
"""Beautiful Soup Elixir and Tonic - "The Screen-Scraper's Friend".
|
||||||
|
|
||||||
|
http://www.crummy.com/software/BeautifulSoup/
|
||||||
|
|
||||||
|
Beautiful Soup uses a pluggable XML or HTML parser to parse a
|
||||||
|
(possibly invalid) document into a tree representation. Beautiful Soup
|
||||||
|
provides methods and Pythonic idioms that make it easy to navigate,
|
||||||
|
search, and modify the parse tree.
|
||||||
|
|
||||||
|
Beautiful Soup works with Python 3.5 and up. It works better if lxml
|
||||||
|
and/or html5lib is installed.
|
||||||
|
|
||||||
|
For more than you ever wanted to know about Beautiful Soup, see the
|
||||||
|
documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/
|
||||||
|
"""
|
||||||
|
|
||||||
|
__author__ = "Leonard Richardson (leonardr@segfault.org)"
|
||||||
|
__version__ = "4.11.1"
|
||||||
|
__copyright__ = "Copyright (c) 2004-2022 Leonard Richardson"
|
||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
__all__ = ['BeautifulSoup']
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
# The very first thing we do is give a useful error if someone is
|
||||||
|
# running this code under Python 2.
|
||||||
|
if sys.version_info.major < 3:
|
||||||
|
raise ImportError('You are trying to use a Python 3-specific version of Beautiful Soup under Python 2. This will not work. The final version of Beautiful Soup to support Python 2 was 4.9.3.')
|
||||||
|
|
||||||
|
from .builder import (
|
||||||
|
builder_registry,
|
||||||
|
ParserRejectedMarkup,
|
||||||
|
XMLParsedAsHTMLWarning,
|
||||||
|
)
|
||||||
|
from .dammit import UnicodeDammit
|
||||||
|
from .element import (
|
||||||
|
CData,
|
||||||
|
Comment,
|
||||||
|
DEFAULT_OUTPUT_ENCODING,
|
||||||
|
Declaration,
|
||||||
|
Doctype,
|
||||||
|
NavigableString,
|
||||||
|
PageElement,
|
||||||
|
ProcessingInstruction,
|
||||||
|
PYTHON_SPECIFIC_ENCODINGS,
|
||||||
|
ResultSet,
|
||||||
|
Script,
|
||||||
|
Stylesheet,
|
||||||
|
SoupStrainer,
|
||||||
|
Tag,
|
||||||
|
TemplateString,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Define some custom warnings.
|
||||||
|
class GuessedAtParserWarning(UserWarning):
|
||||||
|
"""The warning issued when BeautifulSoup has to guess what parser to
|
||||||
|
use -- probably because no parser was specified in the constructor.
|
||||||
|
"""
|
||||||
|
|
||||||
|
class MarkupResemblesLocatorWarning(UserWarning):
|
||||||
|
"""The warning issued when BeautifulSoup is given 'markup' that
|
||||||
|
actually looks like a resource locator -- a URL or a path to a file
|
||||||
|
on disk.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class BeautifulSoup(Tag):
|
||||||
|
"""A data structure representing a parsed HTML or XML document.
|
||||||
|
|
||||||
|
Most of the methods you'll call on a BeautifulSoup object are inherited from
|
||||||
|
PageElement or Tag.
|
||||||
|
|
||||||
|
Internally, this class defines the basic interface called by the
|
||||||
|
tree builders when converting an HTML/XML document into a data
|
||||||
|
structure. The interface abstracts away the differences between
|
||||||
|
parsers. To write a new tree builder, you'll need to understand
|
||||||
|
these methods as a whole.
|
||||||
|
|
||||||
|
These methods will be called by the BeautifulSoup constructor:
|
||||||
|
* reset()
|
||||||
|
* feed(markup)
|
||||||
|
|
||||||
|
The tree builder may call these methods from its feed() implementation:
|
||||||
|
* handle_starttag(name, attrs) # See note about return value
|
||||||
|
* handle_endtag(name)
|
||||||
|
* handle_data(data) # Appends to the current data node
|
||||||
|
* endData(containerClass) # Ends the current data node
|
||||||
|
|
||||||
|
No matter how complicated the underlying parser is, you should be
|
||||||
|
able to build a tree using 'start tag' events, 'end tag' events,
|
||||||
|
'data' events, and "done with data" events.
|
||||||
|
|
||||||
|
If you encounter an empty-element tag (aka a self-closing tag,
|
||||||
|
like HTML's <br> tag), call handle_starttag and then
|
||||||
|
handle_endtag.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Since BeautifulSoup subclasses Tag, it's possible to treat it as
|
||||||
|
# a Tag with a .name. This name makes it clear the BeautifulSoup
|
||||||
|
# object isn't a real markup tag.
|
||||||
|
ROOT_TAG_NAME = '[document]'
|
||||||
|
|
||||||
|
# If the end-user gives no indication which tree builder they
|
||||||
|
# want, look for one with these features.
|
||||||
|
DEFAULT_BUILDER_FEATURES = ['html', 'fast']
|
||||||
|
|
||||||
|
# A string containing all ASCII whitespace characters, used in
|
||||||
|
# endData() to detect data chunks that seem 'empty'.
|
||||||
|
ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
|
||||||
|
|
||||||
|
NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
|
||||||
|
|
||||||
|
def __init__(self, markup="", features=None, builder=None,
|
||||||
|
parse_only=None, from_encoding=None, exclude_encodings=None,
|
||||||
|
element_classes=None, **kwargs):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param markup: A string or a file-like object representing
|
||||||
|
markup to be parsed.
|
||||||
|
|
||||||
|
:param features: Desirable features of the parser to be
|
||||||
|
used. This may be the name of a specific parser ("lxml",
|
||||||
|
"lxml-xml", "html.parser", or "html5lib") or it may be the
|
||||||
|
type of markup to be used ("html", "html5", "xml"). It's
|
||||||
|
recommended that you name a specific parser, so that
|
||||||
|
Beautiful Soup gives you the same results across platforms
|
||||||
|
and virtual environments.
|
||||||
|
|
||||||
|
:param builder: A TreeBuilder subclass to instantiate (or
|
||||||
|
instance to use) instead of looking one up based on
|
||||||
|
`features`. You only need to use this if you've implemented a
|
||||||
|
custom TreeBuilder.
|
||||||
|
|
||||||
|
:param parse_only: A SoupStrainer. Only parts of the document
|
||||||
|
matching the SoupStrainer will be considered. This is useful
|
||||||
|
when parsing part of a document that would otherwise be too
|
||||||
|
large to fit into memory.
|
||||||
|
|
||||||
|
:param from_encoding: A string indicating the encoding of the
|
||||||
|
document to be parsed. Pass this in if Beautiful Soup is
|
||||||
|
guessing wrongly about the document's encoding.
|
||||||
|
|
||||||
|
:param exclude_encodings: A list of strings indicating
|
||||||
|
encodings known to be wrong. Pass this in if you don't know
|
||||||
|
the document's encoding but you know Beautiful Soup's guess is
|
||||||
|
wrong.
|
||||||
|
|
||||||
|
:param element_classes: A dictionary mapping BeautifulSoup
|
||||||
|
classes like Tag and NavigableString, to other classes you'd
|
||||||
|
like to be instantiated instead as the parse tree is
|
||||||
|
built. This is useful for subclassing Tag or NavigableString
|
||||||
|
to modify default behavior.
|
||||||
|
|
||||||
|
:param kwargs: For backwards compatibility purposes, the
|
||||||
|
constructor accepts certain keyword arguments used in
|
||||||
|
Beautiful Soup 3. None of these arguments do anything in
|
||||||
|
Beautiful Soup 4; they will result in a warning and then be
|
||||||
|
ignored.
|
||||||
|
|
||||||
|
Apart from this, any keyword arguments passed into the
|
||||||
|
BeautifulSoup constructor are propagated to the TreeBuilder
|
||||||
|
constructor. This makes it possible to configure a
|
||||||
|
TreeBuilder by passing in arguments, not just by saying which
|
||||||
|
one to use.
|
||||||
|
"""
|
||||||
|
if 'convertEntities' in kwargs:
|
||||||
|
del kwargs['convertEntities']
|
||||||
|
warnings.warn(
|
||||||
|
"BS4 does not respect the convertEntities argument to the "
|
||||||
|
"BeautifulSoup constructor. Entities are always converted "
|
||||||
|
"to Unicode characters.")
|
||||||
|
|
||||||
|
if 'markupMassage' in kwargs:
|
||||||
|
del kwargs['markupMassage']
|
||||||
|
warnings.warn(
|
||||||
|
"BS4 does not respect the markupMassage argument to the "
|
||||||
|
"BeautifulSoup constructor. The tree builder is responsible "
|
||||||
|
"for any necessary markup massage.")
|
||||||
|
|
||||||
|
if 'smartQuotesTo' in kwargs:
|
||||||
|
del kwargs['smartQuotesTo']
|
||||||
|
warnings.warn(
|
||||||
|
"BS4 does not respect the smartQuotesTo argument to the "
|
||||||
|
"BeautifulSoup constructor. Smart quotes are always converted "
|
||||||
|
"to Unicode characters.")
|
||||||
|
|
||||||
|
if 'selfClosingTags' in kwargs:
|
||||||
|
del kwargs['selfClosingTags']
|
||||||
|
warnings.warn(
|
||||||
|
"BS4 does not respect the selfClosingTags argument to the "
|
||||||
|
"BeautifulSoup constructor. The tree builder is responsible "
|
||||||
|
"for understanding self-closing tags.")
|
||||||
|
|
||||||
|
if 'isHTML' in kwargs:
|
||||||
|
del kwargs['isHTML']
|
||||||
|
warnings.warn(
|
||||||
|
"BS4 does not respect the isHTML argument to the "
|
||||||
|
"BeautifulSoup constructor. Suggest you use "
|
||||||
|
"features='lxml' for HTML and features='lxml-xml' for "
|
||||||
|
"XML.")
|
||||||
|
|
||||||
|
def deprecated_argument(old_name, new_name):
|
||||||
|
if old_name in kwargs:
|
||||||
|
warnings.warn(
|
||||||
|
'The "%s" argument to the BeautifulSoup constructor '
|
||||||
|
'has been renamed to "%s."' % (old_name, new_name),
|
||||||
|
DeprecationWarning
|
||||||
|
)
|
||||||
|
return kwargs.pop(old_name)
|
||||||
|
return None
|
||||||
|
|
||||||
|
parse_only = parse_only or deprecated_argument(
|
||||||
|
"parseOnlyThese", "parse_only")
|
||||||
|
|
||||||
|
from_encoding = from_encoding or deprecated_argument(
|
||||||
|
"fromEncoding", "from_encoding")
|
||||||
|
|
||||||
|
if from_encoding and isinstance(markup, str):
|
||||||
|
warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
|
||||||
|
from_encoding = None
|
||||||
|
|
||||||
|
self.element_classes = element_classes or dict()
|
||||||
|
|
||||||
|
# We need this information to track whether or not the builder
|
||||||
|
# was specified well enough that we can omit the 'you need to
|
||||||
|
# specify a parser' warning.
|
||||||
|
original_builder = builder
|
||||||
|
original_features = features
|
||||||
|
|
||||||
|
if isinstance(builder, type):
|
||||||
|
# A builder class was passed in; it needs to be instantiated.
|
||||||
|
builder_class = builder
|
||||||
|
builder = None
|
||||||
|
elif builder is None:
|
||||||
|
if isinstance(features, str):
|
||||||
|
features = [features]
|
||||||
|
if features is None or len(features) == 0:
|
||||||
|
features = self.DEFAULT_BUILDER_FEATURES
|
||||||
|
builder_class = builder_registry.lookup(*features)
|
||||||
|
if builder_class is None:
|
||||||
|
raise FeatureNotFound(
|
||||||
|
"Couldn't find a tree builder with the features you "
|
||||||
|
"requested: %s. Do you need to install a parser library?"
|
||||||
|
% ",".join(features))
|
||||||
|
|
||||||
|
# At this point either we have a TreeBuilder instance in
|
||||||
|
# builder, or we have a builder_class that we can instantiate
|
||||||
|
# with the remaining **kwargs.
|
||||||
|
if builder is None:
|
||||||
|
builder = builder_class(**kwargs)
|
||||||
|
if not original_builder and not (
|
||||||
|
original_features == builder.NAME or
|
||||||
|
original_features in builder.ALTERNATE_NAMES
|
||||||
|
) and markup:
|
||||||
|
# The user did not tell us which TreeBuilder to use,
|
||||||
|
# and we had to guess. Issue a warning.
|
||||||
|
if builder.is_xml:
|
||||||
|
markup_type = "XML"
|
||||||
|
else:
|
||||||
|
markup_type = "HTML"
|
||||||
|
|
||||||
|
# This code adapted from warnings.py so that we get the same line
|
||||||
|
# of code as our warnings.warn() call gets, even if the answer is wrong
|
||||||
|
# (as it may be in a multithreading situation).
|
||||||
|
caller = None
|
||||||
|
try:
|
||||||
|
caller = sys._getframe(1)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
if caller:
|
||||||
|
globals = caller.f_globals
|
||||||
|
line_number = caller.f_lineno
|
||||||
|
else:
|
||||||
|
globals = sys.__dict__
|
||||||
|
line_number= 1
|
||||||
|
filename = globals.get('__file__')
|
||||||
|
if filename:
|
||||||
|
fnl = filename.lower()
|
||||||
|
if fnl.endswith((".pyc", ".pyo")):
|
||||||
|
filename = filename[:-1]
|
||||||
|
if filename:
|
||||||
|
# If there is no filename at all, the user is most likely in a REPL,
|
||||||
|
# and the warning is not necessary.
|
||||||
|
values = dict(
|
||||||
|
filename=filename,
|
||||||
|
line_number=line_number,
|
||||||
|
parser=builder.NAME,
|
||||||
|
markup_type=markup_type
|
||||||
|
)
|
||||||
|
warnings.warn(
|
||||||
|
self.NO_PARSER_SPECIFIED_WARNING % values,
|
||||||
|
GuessedAtParserWarning, stacklevel=2
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if kwargs:
|
||||||
|
warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
|
||||||
|
|
||||||
|
self.builder = builder
|
||||||
|
self.is_xml = builder.is_xml
|
||||||
|
self.known_xml = self.is_xml
|
||||||
|
self._namespaces = dict()
|
||||||
|
self.parse_only = parse_only
|
||||||
|
|
||||||
|
if hasattr(markup, 'read'): # It's a file-type object.
|
||||||
|
markup = markup.read()
|
||||||
|
elif len(markup) <= 256 and (
|
||||||
|
(isinstance(markup, bytes) and not b'<' in markup)
|
||||||
|
or (isinstance(markup, str) and not '<' in markup)
|
||||||
|
):
|
||||||
|
# Issue warnings for a couple beginner problems
|
||||||
|
# involving passing non-markup to Beautiful Soup.
|
||||||
|
# Beautiful Soup will still parse the input as markup,
|
||||||
|
# since that is sometimes the intended behavior.
|
||||||
|
if not self._markup_is_url(markup):
|
||||||
|
self._markup_resembles_filename(markup)
|
||||||
|
|
||||||
|
rejections = []
|
||||||
|
success = False
|
||||||
|
for (self.markup, self.original_encoding, self.declared_html_encoding,
|
||||||
|
self.contains_replacement_characters) in (
|
||||||
|
self.builder.prepare_markup(
|
||||||
|
markup, from_encoding, exclude_encodings=exclude_encodings)):
|
||||||
|
self.reset()
|
||||||
|
self.builder.initialize_soup(self)
|
||||||
|
try:
|
||||||
|
self._feed()
|
||||||
|
success = True
|
||||||
|
break
|
||||||
|
except ParserRejectedMarkup as e:
|
||||||
|
rejections.append(e)
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not success:
|
||||||
|
other_exceptions = [str(e) for e in rejections]
|
||||||
|
raise ParserRejectedMarkup(
|
||||||
|
"The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Clear out the markup and remove the builder's circular
|
||||||
|
# reference to this object.
|
||||||
|
self.markup = None
|
||||||
|
self.builder.soup = None
|
||||||
|
|
||||||
|
def __copy__(self):
|
||||||
|
"""Copy a BeautifulSoup object by converting the document to a string and parsing it again."""
|
||||||
|
copy = type(self)(
|
||||||
|
self.encode('utf-8'), builder=self.builder, from_encoding='utf-8'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Although we encoded the tree to UTF-8, that may not have
|
||||||
|
# been the encoding of the original markup. Set the copy's
|
||||||
|
# .original_encoding to reflect the original object's
|
||||||
|
# .original_encoding.
|
||||||
|
copy.original_encoding = self.original_encoding
|
||||||
|
return copy
|
||||||
|
|
||||||
|
def __getstate__(self):
|
||||||
|
# Frequently a tree builder can't be pickled.
|
||||||
|
d = dict(self.__dict__)
|
||||||
|
if 'builder' in d and d['builder'] is not None and not self.builder.picklable:
|
||||||
|
d['builder'] = None
|
||||||
|
return d
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _decode_markup(cls, markup):
|
||||||
|
"""Ensure `markup` is bytes so it's safe to send into warnings.warn.
|
||||||
|
|
||||||
|
TODO: warnings.warn had this problem back in 2010 but it might not
|
||||||
|
anymore.
|
||||||
|
"""
|
||||||
|
if isinstance(markup, bytes):
|
||||||
|
decoded = markup.decode('utf-8', 'replace')
|
||||||
|
else:
|
||||||
|
decoded = markup
|
||||||
|
return decoded
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _markup_is_url(cls, markup):
|
||||||
|
"""Error-handling method to raise a warning if incoming markup looks
|
||||||
|
like a URL.
|
||||||
|
|
||||||
|
:param markup: A string.
|
||||||
|
:return: Whether or not the markup resembles a URL
|
||||||
|
closely enough to justify a warning.
|
||||||
|
"""
|
||||||
|
if isinstance(markup, bytes):
|
||||||
|
space = b' '
|
||||||
|
cant_start_with = (b"http:", b"https:")
|
||||||
|
elif isinstance(markup, str):
|
||||||
|
space = ' '
|
||||||
|
cant_start_with = ("http:", "https:")
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if any(markup.startswith(prefix) for prefix in cant_start_with):
|
||||||
|
if not space in markup:
|
||||||
|
warnings.warn(
|
||||||
|
'The input looks more like a URL than markup. You may want to use'
|
||||||
|
' an HTTP client like requests to get the document behind'
|
||||||
|
' the URL, and feed that document to Beautiful Soup.',
|
||||||
|
MarkupResemblesLocatorWarning
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _markup_resembles_filename(cls, markup):
|
||||||
|
"""Error-handling method to raise a warning if incoming markup
|
||||||
|
resembles a filename.
|
||||||
|
|
||||||
|
:param markup: A bytestring or string.
|
||||||
|
:return: Whether or not the markup resembles a filename
|
||||||
|
closely enough to justify a warning.
|
||||||
|
"""
|
||||||
|
path_characters = '/\\'
|
||||||
|
extensions = ['.html', '.htm', '.xml', '.xhtml', '.txt']
|
||||||
|
if isinstance(markup, bytes):
|
||||||
|
path_characters = path_characters.encode("utf8")
|
||||||
|
extensions = [x.encode('utf8') for x in extensions]
|
||||||
|
filelike = False
|
||||||
|
if any(x in markup for x in path_characters):
|
||||||
|
filelike = True
|
||||||
|
else:
|
||||||
|
lower = markup.lower()
|
||||||
|
if any(lower.endswith(ext) for ext in extensions):
|
||||||
|
filelike = True
|
||||||
|
if filelike:
|
||||||
|
warnings.warn(
|
||||||
|
'The input looks more like a filename than markup. You may'
|
||||||
|
' want to open this file and pass the filehandle into'
|
||||||
|
' Beautiful Soup.',
|
||||||
|
MarkupResemblesLocatorWarning
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _feed(self):
|
||||||
|
"""Internal method that parses previously set markup, creating a large
|
||||||
|
number of Tag and NavigableString objects.
|
||||||
|
"""
|
||||||
|
# Convert the document to Unicode.
|
||||||
|
self.builder.reset()
|
||||||
|
|
||||||
|
self.builder.feed(self.markup)
|
||||||
|
# Close out any unfinished strings and close all the open tags.
|
||||||
|
self.endData()
|
||||||
|
while self.currentTag.name != self.ROOT_TAG_NAME:
|
||||||
|
self.popTag()
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
"""Reset this object to a state as though it had never parsed any
|
||||||
|
markup.
|
||||||
|
"""
|
||||||
|
Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
|
||||||
|
self.hidden = 1
|
||||||
|
self.builder.reset()
|
||||||
|
self.current_data = []
|
||||||
|
self.currentTag = None
|
||||||
|
self.tagStack = []
|
||||||
|
self.open_tag_counter = Counter()
|
||||||
|
self.preserve_whitespace_tag_stack = []
|
||||||
|
self.string_container_stack = []
|
||||||
|
self.pushTag(self)
|
||||||
|
|
||||||
|
def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
|
||||||
|
sourceline=None, sourcepos=None, **kwattrs):
|
||||||
|
"""Create a new Tag associated with this BeautifulSoup object.
|
||||||
|
|
||||||
|
:param name: The name of the new Tag.
|
||||||
|
:param namespace: The URI of the new Tag's XML namespace, if any.
|
||||||
|
:param prefix: The prefix for the new Tag's XML namespace, if any.
|
||||||
|
:param attrs: A dictionary of this Tag's attribute values; can
|
||||||
|
be used instead of `kwattrs` for attributes like 'class'
|
||||||
|
that are reserved words in Python.
|
||||||
|
:param sourceline: The line number where this tag was
|
||||||
|
(purportedly) found in its source document.
|
||||||
|
:param sourcepos: The character position within `sourceline` where this
|
||||||
|
tag was (purportedly) found.
|
||||||
|
:param kwattrs: Keyword arguments for the new Tag's attribute values.
|
||||||
|
|
||||||
|
"""
|
||||||
|
kwattrs.update(attrs)
|
||||||
|
return self.element_classes.get(Tag, Tag)(
|
||||||
|
None, self.builder, name, namespace, nsprefix, kwattrs,
|
||||||
|
sourceline=sourceline, sourcepos=sourcepos
|
||||||
|
)
|
||||||
|
|
||||||
|
def string_container(self, base_class=None):
|
||||||
|
container = base_class or NavigableString
|
||||||
|
|
||||||
|
# There may be a general override of NavigableString.
|
||||||
|
container = self.element_classes.get(
|
||||||
|
container, container
|
||||||
|
)
|
||||||
|
|
||||||
|
# On top of that, we may be inside a tag that needs a special
|
||||||
|
# container class.
|
||||||
|
if self.string_container_stack and container is NavigableString:
|
||||||
|
container = self.builder.string_containers.get(
|
||||||
|
self.string_container_stack[-1].name, container
|
||||||
|
)
|
||||||
|
return container
|
||||||
|
|
||||||
|
def new_string(self, s, subclass=None):
|
||||||
|
"""Create a new NavigableString associated with this BeautifulSoup
|
||||||
|
object.
|
||||||
|
"""
|
||||||
|
container = self.string_container(subclass)
|
||||||
|
return container(s)
|
||||||
|
|
||||||
|
def insert_before(self, *args):
|
||||||
|
"""This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
|
||||||
|
it because there is nothing before or after it in the parse tree.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
|
||||||
|
|
||||||
|
def insert_after(self, *args):
|
||||||
|
"""This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
|
||||||
|
it because there is nothing before or after it in the parse tree.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
|
||||||
|
|
||||||
|
def popTag(self):
|
||||||
|
"""Internal method called by _popToTag when a tag is closed."""
|
||||||
|
tag = self.tagStack.pop()
|
||||||
|
if tag.name in self.open_tag_counter:
|
||||||
|
self.open_tag_counter[tag.name] -= 1
|
||||||
|
if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
|
||||||
|
self.preserve_whitespace_tag_stack.pop()
|
||||||
|
if self.string_container_stack and tag == self.string_container_stack[-1]:
|
||||||
|
self.string_container_stack.pop()
|
||||||
|
#print("Pop", tag.name)
|
||||||
|
if self.tagStack:
|
||||||
|
self.currentTag = self.tagStack[-1]
|
||||||
|
return self.currentTag
|
||||||
|
|
||||||
|
def pushTag(self, tag):
|
||||||
|
"""Internal method called by handle_starttag when a tag is opened."""
|
||||||
|
#print("Push", tag.name)
|
||||||
|
if self.currentTag is not None:
|
||||||
|
self.currentTag.contents.append(tag)
|
||||||
|
self.tagStack.append(tag)
|
||||||
|
self.currentTag = self.tagStack[-1]
|
||||||
|
if tag.name != self.ROOT_TAG_NAME:
|
||||||
|
self.open_tag_counter[tag.name] += 1
|
||||||
|
if tag.name in self.builder.preserve_whitespace_tags:
|
||||||
|
self.preserve_whitespace_tag_stack.append(tag)
|
||||||
|
if tag.name in self.builder.string_containers:
|
||||||
|
self.string_container_stack.append(tag)
|
||||||
|
|
||||||
|
def endData(self, containerClass=None):
|
||||||
|
"""Method called by the TreeBuilder when the end of a data segment
|
||||||
|
occurs.
|
||||||
|
"""
|
||||||
|
if self.current_data:
|
||||||
|
current_data = ''.join(self.current_data)
|
||||||
|
# If whitespace is not preserved, and this string contains
|
||||||
|
# nothing but ASCII spaces, replace it with a single space
|
||||||
|
# or newline.
|
||||||
|
if not self.preserve_whitespace_tag_stack:
|
||||||
|
strippable = True
|
||||||
|
for i in current_data:
|
||||||
|
if i not in self.ASCII_SPACES:
|
||||||
|
strippable = False
|
||||||
|
break
|
||||||
|
if strippable:
|
||||||
|
if '\n' in current_data:
|
||||||
|
current_data = '\n'
|
||||||
|
else:
|
||||||
|
current_data = ' '
|
||||||
|
|
||||||
|
# Reset the data collector.
|
||||||
|
self.current_data = []
|
||||||
|
|
||||||
|
# Should we add this string to the tree at all?
|
||||||
|
if self.parse_only and len(self.tagStack) <= 1 and \
|
||||||
|
(not self.parse_only.text or \
|
||||||
|
not self.parse_only.search(current_data)):
|
||||||
|
return
|
||||||
|
|
||||||
|
containerClass = self.string_container(containerClass)
|
||||||
|
o = containerClass(current_data)
|
||||||
|
self.object_was_parsed(o)
|
||||||
|
|
||||||
|
def object_was_parsed(self, o, parent=None, most_recent_element=None):
|
||||||
|
"""Method called by the TreeBuilder to integrate an object into the parse tree."""
|
||||||
|
if parent is None:
|
||||||
|
parent = self.currentTag
|
||||||
|
if most_recent_element is not None:
|
||||||
|
previous_element = most_recent_element
|
||||||
|
else:
|
||||||
|
previous_element = self._most_recent_element
|
||||||
|
|
||||||
|
next_element = previous_sibling = next_sibling = None
|
||||||
|
if isinstance(o, Tag):
|
||||||
|
next_element = o.next_element
|
||||||
|
next_sibling = o.next_sibling
|
||||||
|
previous_sibling = o.previous_sibling
|
||||||
|
if previous_element is None:
|
||||||
|
previous_element = o.previous_element
|
||||||
|
|
||||||
|
fix = parent.next_element is not None
|
||||||
|
|
||||||
|
o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
|
||||||
|
|
||||||
|
self._most_recent_element = o
|
||||||
|
parent.contents.append(o)
|
||||||
|
|
||||||
|
# Check if we are inserting into an already parsed node.
|
||||||
|
if fix:
|
||||||
|
self._linkage_fixer(parent)
|
||||||
|
|
||||||
|
def _linkage_fixer(self, el):
|
||||||
|
"""Make sure linkage of this fragment is sound."""
|
||||||
|
|
||||||
|
first = el.contents[0]
|
||||||
|
child = el.contents[-1]
|
||||||
|
descendant = child
|
||||||
|
|
||||||
|
if child is first and el.parent is not None:
|
||||||
|
# Parent should be linked to first child
|
||||||
|
el.next_element = child
|
||||||
|
# We are no longer linked to whatever this element is
|
||||||
|
prev_el = child.previous_element
|
||||||
|
if prev_el is not None and prev_el is not el:
|
||||||
|
prev_el.next_element = None
|
||||||
|
# First child should be linked to the parent, and no previous siblings.
|
||||||
|
child.previous_element = el
|
||||||
|
child.previous_sibling = None
|
||||||
|
|
||||||
|
# We have no sibling as we've been appended as the last.
|
||||||
|
child.next_sibling = None
|
||||||
|
|
||||||
|
# This index is a tag, dig deeper for a "last descendant"
|
||||||
|
if isinstance(child, Tag) and child.contents:
|
||||||
|
descendant = child._last_descendant(False)
|
||||||
|
|
||||||
|
# As the final step, link last descendant. It should be linked
|
||||||
|
# to the parent's next sibling (if found), else walk up the chain
|
||||||
|
# and find a parent with a sibling. It should have no next sibling.
|
||||||
|
descendant.next_element = None
|
||||||
|
descendant.next_sibling = None
|
||||||
|
target = el
|
||||||
|
while True:
|
||||||
|
if target is None:
|
||||||
|
break
|
||||||
|
elif target.next_sibling is not None:
|
||||||
|
descendant.next_element = target.next_sibling
|
||||||
|
target.next_sibling.previous_element = child
|
||||||
|
break
|
||||||
|
target = target.parent
|
||||||
|
|
||||||
|
def _popToTag(self, name, nsprefix=None, inclusivePop=True):
|
||||||
|
"""Pops the tag stack up to and including the most recent
|
||||||
|
instance of the given tag.
|
||||||
|
|
||||||
|
If there are no open tags with the given name, nothing will be
|
||||||
|
popped.
|
||||||
|
|
||||||
|
:param name: Pop up to the most recent tag with this name.
|
||||||
|
:param nsprefix: The namespace prefix that goes with `name`.
|
||||||
|
:param inclusivePop: It this is false, pops the tag stack up
|
||||||
|
to but *not* including the most recent instqance of the
|
||||||
|
given tag.
|
||||||
|
|
||||||
|
"""
|
||||||
|
#print("Popping to %s" % name)
|
||||||
|
if name == self.ROOT_TAG_NAME:
|
||||||
|
# The BeautifulSoup object itself can never be popped.
|
||||||
|
return
|
||||||
|
|
||||||
|
most_recently_popped = None
|
||||||
|
|
||||||
|
stack_size = len(self.tagStack)
|
||||||
|
for i in range(stack_size - 1, 0, -1):
|
||||||
|
if not self.open_tag_counter.get(name):
|
||||||
|
break
|
||||||
|
t = self.tagStack[i]
|
||||||
|
if (name == t.name and nsprefix == t.prefix):
|
||||||
|
if inclusivePop:
|
||||||
|
most_recently_popped = self.popTag()
|
||||||
|
break
|
||||||
|
most_recently_popped = self.popTag()
|
||||||
|
|
||||||
|
return most_recently_popped
|
||||||
|
|
||||||
|
def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
|
||||||
|
sourcepos=None, namespaces=None):
|
||||||
|
"""Called by the tree builder when a new tag is encountered.
|
||||||
|
|
||||||
|
:param name: Name of the tag.
|
||||||
|
:param nsprefix: Namespace prefix for the tag.
|
||||||
|
:param attrs: A dictionary of attribute values.
|
||||||
|
:param sourceline: The line number where this tag was found in its
|
||||||
|
source document.
|
||||||
|
:param sourcepos: The character position within `sourceline` where this
|
||||||
|
tag was found.
|
||||||
|
:param namespaces: A dictionary of all namespace prefix mappings
|
||||||
|
currently in scope in the document.
|
||||||
|
|
||||||
|
If this method returns None, the tag was rejected by an active
|
||||||
|
SoupStrainer. You should proceed as if the tag had not occurred
|
||||||
|
in the document. For instance, if this was a self-closing tag,
|
||||||
|
don't call handle_endtag.
|
||||||
|
"""
|
||||||
|
# print("Start tag %s: %s" % (name, attrs))
|
||||||
|
self.endData()
|
||||||
|
|
||||||
|
if (self.parse_only and len(self.tagStack) <= 1
|
||||||
|
and (self.parse_only.text
|
||||||
|
or not self.parse_only.search_tag(name, attrs))):
|
||||||
|
return None
|
||||||
|
|
||||||
|
tag = self.element_classes.get(Tag, Tag)(
|
||||||
|
self, self.builder, name, namespace, nsprefix, attrs,
|
||||||
|
self.currentTag, self._most_recent_element,
|
||||||
|
sourceline=sourceline, sourcepos=sourcepos,
|
||||||
|
namespaces=namespaces
|
||||||
|
)
|
||||||
|
if tag is None:
|
||||||
|
return tag
|
||||||
|
if self._most_recent_element is not None:
|
||||||
|
self._most_recent_element.next_element = tag
|
||||||
|
self._most_recent_element = tag
|
||||||
|
self.pushTag(tag)
|
||||||
|
return tag
|
||||||
|
|
||||||
|
def handle_endtag(self, name, nsprefix=None):
|
||||||
|
"""Called by the tree builder when an ending tag is encountered.
|
||||||
|
|
||||||
|
:param name: Name of the tag.
|
||||||
|
:param nsprefix: Namespace prefix for the tag.
|
||||||
|
"""
|
||||||
|
#print("End tag: " + name)
|
||||||
|
self.endData()
|
||||||
|
self._popToTag(name, nsprefix)
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
"""Called by the tree builder when a chunk of textual data is encountered."""
|
||||||
|
self.current_data.append(data)
|
||||||
|
|
||||||
|
def decode(self, pretty_print=False,
|
||||||
|
eventual_encoding=DEFAULT_OUTPUT_ENCODING,
|
||||||
|
formatter="minimal"):
|
||||||
|
"""Returns a string or Unicode representation of the parse tree
|
||||||
|
as an HTML or XML document.
|
||||||
|
|
||||||
|
:param pretty_print: If this is True, indentation will be used to
|
||||||
|
make the document more readable.
|
||||||
|
:param eventual_encoding: The encoding of the final document.
|
||||||
|
If this is None, the document will be a Unicode string.
|
||||||
|
"""
|
||||||
|
if self.is_xml:
|
||||||
|
# Print the XML declaration
|
||||||
|
encoding_part = ''
|
||||||
|
if eventual_encoding in PYTHON_SPECIFIC_ENCODINGS:
|
||||||
|
# This is a special Python encoding; it can't actually
|
||||||
|
# go into an XML document because it means nothing
|
||||||
|
# outside of Python.
|
||||||
|
eventual_encoding = None
|
||||||
|
if eventual_encoding != None:
|
||||||
|
encoding_part = ' encoding="%s"' % eventual_encoding
|
||||||
|
prefix = '<?xml version="1.0"%s?>\n' % encoding_part
|
||||||
|
else:
|
||||||
|
prefix = ''
|
||||||
|
if not pretty_print:
|
||||||
|
indent_level = None
|
||||||
|
else:
|
||||||
|
indent_level = 0
|
||||||
|
return prefix + super(BeautifulSoup, self).decode(
|
||||||
|
indent_level, eventual_encoding, formatter)
|
||||||
|
|
||||||
|
# Aliases to make it easier to get started quickly, e.g. 'from bs4 import _soup'
|
||||||
|
_s = BeautifulSoup
|
||||||
|
_soup = BeautifulSoup
|
||||||
|
|
||||||
|
class BeautifulStoneSoup(BeautifulSoup):
|
||||||
|
"""Deprecated interface to an XML parser."""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
kwargs['features'] = 'xml'
|
||||||
|
warnings.warn(
|
||||||
|
'The BeautifulStoneSoup class is deprecated. Instead of using '
|
||||||
|
'it, pass features="xml" into the BeautifulSoup constructor.',
|
||||||
|
DeprecationWarning
|
||||||
|
)
|
||||||
|
super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class StopParsing(Exception):
|
||||||
|
"""Exception raised by a TreeBuilder if it's unable to continue parsing."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
class FeatureNotFound(ValueError):
|
||||||
|
"""Exception raised by the BeautifulSoup constructor if no parser with the
|
||||||
|
requested features is found.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
#If this file is run as a script, act as an HTML pretty-printer.
|
||||||
|
if __name__ == '__main__':
|
||||||
|
import sys
|
||||||
|
soup = BeautifulSoup(sys.stdin)
|
||||||
|
print((soup.prettify()))
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
631
acer-env/lib/python3.10/site-packages/bs4/builder/__init__.py
Normal file
631
acer-env/lib/python3.10/site-packages/bs4/builder/__init__.py
Normal file
@ -0,0 +1,631 @@
|
|||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
import itertools
|
||||||
|
import re
|
||||||
|
import warnings
|
||||||
|
import sys
|
||||||
|
from bs4.element import (
|
||||||
|
CharsetMetaAttributeValue,
|
||||||
|
ContentMetaAttributeValue,
|
||||||
|
RubyParenthesisString,
|
||||||
|
RubyTextString,
|
||||||
|
Stylesheet,
|
||||||
|
Script,
|
||||||
|
TemplateString,
|
||||||
|
nonwhitespace_re
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'HTMLTreeBuilder',
|
||||||
|
'SAXTreeBuilder',
|
||||||
|
'TreeBuilder',
|
||||||
|
'TreeBuilderRegistry',
|
||||||
|
]
|
||||||
|
|
||||||
|
# Some useful features for a TreeBuilder to have.
|
||||||
|
FAST = 'fast'
|
||||||
|
PERMISSIVE = 'permissive'
|
||||||
|
STRICT = 'strict'
|
||||||
|
XML = 'xml'
|
||||||
|
HTML = 'html'
|
||||||
|
HTML_5 = 'html5'
|
||||||
|
|
||||||
|
class XMLParsedAsHTMLWarning(UserWarning):
|
||||||
|
"""The warning issued when an HTML parser is used to parse
|
||||||
|
XML that is not XHTML.
|
||||||
|
"""
|
||||||
|
MESSAGE = """It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor."""
|
||||||
|
|
||||||
|
|
||||||
|
class TreeBuilderRegistry(object):
|
||||||
|
"""A way of looking up TreeBuilder subclasses by their name or by desired
|
||||||
|
features.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.builders_for_feature = defaultdict(list)
|
||||||
|
self.builders = []
|
||||||
|
|
||||||
|
def register(self, treebuilder_class):
|
||||||
|
"""Register a treebuilder based on its advertised features.
|
||||||
|
|
||||||
|
:param treebuilder_class: A subclass of Treebuilder. its .features
|
||||||
|
attribute should list its features.
|
||||||
|
"""
|
||||||
|
for feature in treebuilder_class.features:
|
||||||
|
self.builders_for_feature[feature].insert(0, treebuilder_class)
|
||||||
|
self.builders.insert(0, treebuilder_class)
|
||||||
|
|
||||||
|
def lookup(self, *features):
|
||||||
|
"""Look up a TreeBuilder subclass with the desired features.
|
||||||
|
|
||||||
|
:param features: A list of features to look for. If none are
|
||||||
|
provided, the most recently registered TreeBuilder subclass
|
||||||
|
will be used.
|
||||||
|
:return: A TreeBuilder subclass, or None if there's no
|
||||||
|
registered subclass with all the requested features.
|
||||||
|
"""
|
||||||
|
if len(self.builders) == 0:
|
||||||
|
# There are no builders at all.
|
||||||
|
return None
|
||||||
|
|
||||||
|
if len(features) == 0:
|
||||||
|
# They didn't ask for any features. Give them the most
|
||||||
|
# recently registered builder.
|
||||||
|
return self.builders[0]
|
||||||
|
|
||||||
|
# Go down the list of features in order, and eliminate any builders
|
||||||
|
# that don't match every feature.
|
||||||
|
features = list(features)
|
||||||
|
features.reverse()
|
||||||
|
candidates = None
|
||||||
|
candidate_set = None
|
||||||
|
while len(features) > 0:
|
||||||
|
feature = features.pop()
|
||||||
|
we_have_the_feature = self.builders_for_feature.get(feature, [])
|
||||||
|
if len(we_have_the_feature) > 0:
|
||||||
|
if candidates is None:
|
||||||
|
candidates = we_have_the_feature
|
||||||
|
candidate_set = set(candidates)
|
||||||
|
else:
|
||||||
|
# Eliminate any candidates that don't have this feature.
|
||||||
|
candidate_set = candidate_set.intersection(
|
||||||
|
set(we_have_the_feature))
|
||||||
|
|
||||||
|
# The only valid candidates are the ones in candidate_set.
|
||||||
|
# Go through the original list of candidates and pick the first one
|
||||||
|
# that's in candidate_set.
|
||||||
|
if candidate_set is None:
|
||||||
|
return None
|
||||||
|
for candidate in candidates:
|
||||||
|
if candidate in candidate_set:
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
# The BeautifulSoup class will take feature lists from developers and use them
|
||||||
|
# to look up builders in this registry.
|
||||||
|
builder_registry = TreeBuilderRegistry()
|
||||||
|
|
||||||
|
class TreeBuilder(object):
|
||||||
|
"""Turn a textual document into a Beautiful Soup object tree."""
|
||||||
|
|
||||||
|
NAME = "[Unknown tree builder]"
|
||||||
|
ALTERNATE_NAMES = []
|
||||||
|
features = []
|
||||||
|
|
||||||
|
is_xml = False
|
||||||
|
picklable = False
|
||||||
|
empty_element_tags = None # A tag will be considered an empty-element
|
||||||
|
# tag when and only when it has no contents.
|
||||||
|
|
||||||
|
# A value for these tag/attribute combinations is a space- or
|
||||||
|
# comma-separated list of CDATA, rather than a single CDATA.
|
||||||
|
DEFAULT_CDATA_LIST_ATTRIBUTES = {}
|
||||||
|
|
||||||
|
# Whitespace should be preserved inside these tags.
|
||||||
|
DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
|
||||||
|
|
||||||
|
# The textual contents of tags with these names should be
|
||||||
|
# instantiated with some class other than NavigableString.
|
||||||
|
DEFAULT_STRING_CONTAINERS = {}
|
||||||
|
|
||||||
|
USE_DEFAULT = object()
|
||||||
|
|
||||||
|
# Most parsers don't keep track of line numbers.
|
||||||
|
TRACKS_LINE_NUMBERS = False
|
||||||
|
|
||||||
|
def __init__(self, multi_valued_attributes=USE_DEFAULT,
|
||||||
|
preserve_whitespace_tags=USE_DEFAULT,
|
||||||
|
store_line_numbers=USE_DEFAULT,
|
||||||
|
string_containers=USE_DEFAULT,
|
||||||
|
):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param multi_valued_attributes: If this is set to None, the
|
||||||
|
TreeBuilder will not turn any values for attributes like
|
||||||
|
'class' into lists. Setting this to a dictionary will
|
||||||
|
customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
|
||||||
|
for an example.
|
||||||
|
|
||||||
|
Internally, these are called "CDATA list attributes", but that
|
||||||
|
probably doesn't make sense to an end-user, so the argument name
|
||||||
|
is `multi_valued_attributes`.
|
||||||
|
|
||||||
|
:param preserve_whitespace_tags: A list of tags to treat
|
||||||
|
the way <pre> tags are treated in HTML. Tags in this list
|
||||||
|
are immune from pretty-printing; their contents will always be
|
||||||
|
output as-is.
|
||||||
|
|
||||||
|
:param string_containers: A dictionary mapping tag names to
|
||||||
|
the classes that should be instantiated to contain the textual
|
||||||
|
contents of those tags. The default is to use NavigableString
|
||||||
|
for every tag, no matter what the name. You can override the
|
||||||
|
default by changing DEFAULT_STRING_CONTAINERS.
|
||||||
|
|
||||||
|
:param store_line_numbers: If the parser keeps track of the
|
||||||
|
line numbers and positions of the original markup, that
|
||||||
|
information will, by default, be stored in each corresponding
|
||||||
|
`Tag` object. You can turn this off by passing
|
||||||
|
store_line_numbers=False. If the parser you're using doesn't
|
||||||
|
keep track of this information, then setting store_line_numbers=True
|
||||||
|
will do nothing.
|
||||||
|
"""
|
||||||
|
self.soup = None
|
||||||
|
if multi_valued_attributes is self.USE_DEFAULT:
|
||||||
|
multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
|
||||||
|
self.cdata_list_attributes = multi_valued_attributes
|
||||||
|
if preserve_whitespace_tags is self.USE_DEFAULT:
|
||||||
|
preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
|
||||||
|
self.preserve_whitespace_tags = preserve_whitespace_tags
|
||||||
|
if store_line_numbers == self.USE_DEFAULT:
|
||||||
|
store_line_numbers = self.TRACKS_LINE_NUMBERS
|
||||||
|
self.store_line_numbers = store_line_numbers
|
||||||
|
if string_containers == self.USE_DEFAULT:
|
||||||
|
string_containers = self.DEFAULT_STRING_CONTAINERS
|
||||||
|
self.string_containers = string_containers
|
||||||
|
|
||||||
|
def initialize_soup(self, soup):
|
||||||
|
"""The BeautifulSoup object has been initialized and is now
|
||||||
|
being associated with the TreeBuilder.
|
||||||
|
|
||||||
|
:param soup: A BeautifulSoup object.
|
||||||
|
"""
|
||||||
|
self.soup = soup
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
"""Do any work necessary to reset the underlying parser
|
||||||
|
for a new document.
|
||||||
|
|
||||||
|
By default, this does nothing.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def can_be_empty_element(self, tag_name):
|
||||||
|
"""Might a tag with this name be an empty-element tag?
|
||||||
|
|
||||||
|
The final markup may or may not actually present this tag as
|
||||||
|
self-closing.
|
||||||
|
|
||||||
|
For instance: an HTMLBuilder does not consider a <p> tag to be
|
||||||
|
an empty-element tag (it's not in
|
||||||
|
HTMLBuilder.empty_element_tags). This means an empty <p> tag
|
||||||
|
will be presented as "<p></p>", not "<p/>" or "<p>".
|
||||||
|
|
||||||
|
The default implementation has no opinion about which tags are
|
||||||
|
empty-element tags, so a tag will be presented as an
|
||||||
|
empty-element tag if and only if it has no children.
|
||||||
|
"<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
|
||||||
|
be left alone.
|
||||||
|
|
||||||
|
:param tag_name: The name of a markup tag.
|
||||||
|
"""
|
||||||
|
if self.empty_element_tags is None:
|
||||||
|
return True
|
||||||
|
return tag_name in self.empty_element_tags
|
||||||
|
|
||||||
|
def feed(self, markup):
|
||||||
|
"""Run some incoming markup through some parsing process,
|
||||||
|
populating the `BeautifulSoup` object in self.soup.
|
||||||
|
|
||||||
|
This method is not implemented in TreeBuilder; it must be
|
||||||
|
implemented in subclasses.
|
||||||
|
|
||||||
|
:return: None.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||||
|
document_declared_encoding=None, exclude_encodings=None):
|
||||||
|
"""Run any preliminary steps necessary to make incoming markup
|
||||||
|
acceptable to the parser.
|
||||||
|
|
||||||
|
:param markup: Some markup -- probably a bytestring.
|
||||||
|
:param user_specified_encoding: The user asked to try this encoding.
|
||||||
|
:param document_declared_encoding: The markup itself claims to be
|
||||||
|
in this encoding. NOTE: This argument is not used by the
|
||||||
|
calling code and can probably be removed.
|
||||||
|
:param exclude_encodings: The user asked _not_ to try any of
|
||||||
|
these encodings.
|
||||||
|
|
||||||
|
:yield: A series of 4-tuples:
|
||||||
|
(markup, encoding, declared encoding,
|
||||||
|
has undergone character replacement)
|
||||||
|
|
||||||
|
Each 4-tuple represents a strategy for converting the
|
||||||
|
document to Unicode and parsing it. Each strategy will be tried
|
||||||
|
in turn.
|
||||||
|
|
||||||
|
By default, the only strategy is to parse the markup
|
||||||
|
as-is. See `LXMLTreeBuilderForXML` and
|
||||||
|
`HTMLParserTreeBuilder` for implementations that take into
|
||||||
|
account the quirks of particular parsers.
|
||||||
|
"""
|
||||||
|
yield markup, None, None, False
|
||||||
|
|
||||||
|
def test_fragment_to_document(self, fragment):
|
||||||
|
"""Wrap an HTML fragment to make it look like a document.
|
||||||
|
|
||||||
|
Different parsers do this differently. For instance, lxml
|
||||||
|
introduces an empty <head> tag, and html5lib
|
||||||
|
doesn't. Abstracting this away lets us write simple tests
|
||||||
|
which run HTML fragments through the parser and compare the
|
||||||
|
results against other HTML fragments.
|
||||||
|
|
||||||
|
This method should not be used outside of tests.
|
||||||
|
|
||||||
|
:param fragment: A string -- fragment of HTML.
|
||||||
|
:return: A string -- a full HTML document.
|
||||||
|
"""
|
||||||
|
return fragment
|
||||||
|
|
||||||
|
def set_up_substitutions(self, tag):
|
||||||
|
"""Set up any substitutions that will need to be performed on
|
||||||
|
a `Tag` when it's output as a string.
|
||||||
|
|
||||||
|
By default, this does nothing. See `HTMLTreeBuilder` for a
|
||||||
|
case where this is used.
|
||||||
|
|
||||||
|
:param tag: A `Tag`
|
||||||
|
:return: Whether or not a substitution was performed.
|
||||||
|
"""
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _replace_cdata_list_attribute_values(self, tag_name, attrs):
|
||||||
|
"""When an attribute value is associated with a tag that can
|
||||||
|
have multiple values for that attribute, convert the string
|
||||||
|
value to a list of strings.
|
||||||
|
|
||||||
|
Basically, replaces class="foo bar" with class=["foo", "bar"]
|
||||||
|
|
||||||
|
NOTE: This method modifies its input in place.
|
||||||
|
|
||||||
|
:param tag_name: The name of a tag.
|
||||||
|
:param attrs: A dictionary containing the tag's attributes.
|
||||||
|
Any appropriate attribute values will be modified in place.
|
||||||
|
"""
|
||||||
|
if not attrs:
|
||||||
|
return attrs
|
||||||
|
if self.cdata_list_attributes:
|
||||||
|
universal = self.cdata_list_attributes.get('*', [])
|
||||||
|
tag_specific = self.cdata_list_attributes.get(
|
||||||
|
tag_name.lower(), None)
|
||||||
|
for attr in list(attrs.keys()):
|
||||||
|
if attr in universal or (tag_specific and attr in tag_specific):
|
||||||
|
# We have a "class"-type attribute whose string
|
||||||
|
# value is a whitespace-separated list of
|
||||||
|
# values. Split it into a list.
|
||||||
|
value = attrs[attr]
|
||||||
|
if isinstance(value, str):
|
||||||
|
values = nonwhitespace_re.findall(value)
|
||||||
|
else:
|
||||||
|
# html5lib sometimes calls setAttributes twice
|
||||||
|
# for the same tag when rearranging the parse
|
||||||
|
# tree. On the second call the attribute value
|
||||||
|
# here is already a list. If this happens,
|
||||||
|
# leave the value alone rather than trying to
|
||||||
|
# split it again.
|
||||||
|
values = value
|
||||||
|
attrs[attr] = values
|
||||||
|
return attrs
|
||||||
|
|
||||||
|
class SAXTreeBuilder(TreeBuilder):
|
||||||
|
"""A Beautiful Soup treebuilder that listens for SAX events.
|
||||||
|
|
||||||
|
This is not currently used for anything, but it demonstrates
|
||||||
|
how a simple TreeBuilder would work.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def feed(self, markup):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def startElement(self, name, attrs):
|
||||||
|
attrs = dict((key[1], value) for key, value in list(attrs.items()))
|
||||||
|
#print("Start %s, %r" % (name, attrs))
|
||||||
|
self.soup.handle_starttag(name, attrs)
|
||||||
|
|
||||||
|
def endElement(self, name):
|
||||||
|
#print("End %s" % name)
|
||||||
|
self.soup.handle_endtag(name)
|
||||||
|
|
||||||
|
def startElementNS(self, nsTuple, nodeName, attrs):
|
||||||
|
# Throw away (ns, nodeName) for now.
|
||||||
|
self.startElement(nodeName, attrs)
|
||||||
|
|
||||||
|
def endElementNS(self, nsTuple, nodeName):
|
||||||
|
# Throw away (ns, nodeName) for now.
|
||||||
|
self.endElement(nodeName)
|
||||||
|
#handler.endElementNS((ns, node.nodeName), node.nodeName)
|
||||||
|
|
||||||
|
def startPrefixMapping(self, prefix, nodeValue):
|
||||||
|
# Ignore the prefix for now.
|
||||||
|
pass
|
||||||
|
|
||||||
|
def endPrefixMapping(self, prefix):
|
||||||
|
# Ignore the prefix for now.
|
||||||
|
# handler.endPrefixMapping(prefix)
|
||||||
|
pass
|
||||||
|
|
||||||
|
def characters(self, content):
|
||||||
|
self.soup.handle_data(content)
|
||||||
|
|
||||||
|
def startDocument(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def endDocument(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class HTMLTreeBuilder(TreeBuilder):
|
||||||
|
"""This TreeBuilder knows facts about HTML.
|
||||||
|
|
||||||
|
Such as which tags are empty-element tags.
|
||||||
|
"""
|
||||||
|
|
||||||
|
empty_element_tags = set([
|
||||||
|
# These are from HTML5.
|
||||||
|
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
|
||||||
|
|
||||||
|
# These are from earlier versions of HTML and are removed in HTML5.
|
||||||
|
'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
|
||||||
|
])
|
||||||
|
|
||||||
|
# The HTML standard defines these as block-level elements. Beautiful
|
||||||
|
# Soup does not treat these elements differently from other elements,
|
||||||
|
# but it may do so eventually, and this information is available if
|
||||||
|
# you need to use it.
|
||||||
|
block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
|
||||||
|
|
||||||
|
# These HTML tags need special treatment so they can be
|
||||||
|
# represented by a string class other than NavigableString.
|
||||||
|
#
|
||||||
|
# For some of these tags, it's because the HTML standard defines
|
||||||
|
# an unusual content model for them. I made this list by going
|
||||||
|
# through the HTML spec
|
||||||
|
# (https://html.spec.whatwg.org/#metadata-content) and looking for
|
||||||
|
# "metadata content" elements that can contain strings.
|
||||||
|
#
|
||||||
|
# The Ruby tags (<rt> and <rp>) are here despite being normal
|
||||||
|
# "phrasing content" tags, because the content they contain is
|
||||||
|
# qualitatively different from other text in the document, and it
|
||||||
|
# can be useful to be able to distinguish it.
|
||||||
|
#
|
||||||
|
# TODO: Arguably <noscript> could go here but it seems
|
||||||
|
# qualitatively different from the other tags.
|
||||||
|
DEFAULT_STRING_CONTAINERS = {
|
||||||
|
'rt' : RubyTextString,
|
||||||
|
'rp' : RubyParenthesisString,
|
||||||
|
'style': Stylesheet,
|
||||||
|
'script': Script,
|
||||||
|
'template': TemplateString,
|
||||||
|
}
|
||||||
|
|
||||||
|
# The HTML standard defines these attributes as containing a
|
||||||
|
# space-separated list of values, not a single value. That is,
|
||||||
|
# class="foo bar" means that the 'class' attribute has two values,
|
||||||
|
# 'foo' and 'bar', not the single value 'foo bar'. When we
|
||||||
|
# encounter one of these attributes, we will parse its value into
|
||||||
|
# a list of values if possible. Upon output, the list will be
|
||||||
|
# converted back into a string.
|
||||||
|
DEFAULT_CDATA_LIST_ATTRIBUTES = {
|
||||||
|
"*" : ['class', 'accesskey', 'dropzone'],
|
||||||
|
"a" : ['rel', 'rev'],
|
||||||
|
"link" : ['rel', 'rev'],
|
||||||
|
"td" : ["headers"],
|
||||||
|
"th" : ["headers"],
|
||||||
|
"td" : ["headers"],
|
||||||
|
"form" : ["accept-charset"],
|
||||||
|
"object" : ["archive"],
|
||||||
|
|
||||||
|
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
|
||||||
|
"area" : ["rel"],
|
||||||
|
"icon" : ["sizes"],
|
||||||
|
"iframe" : ["sandbox"],
|
||||||
|
"output" : ["for"],
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
|
||||||
|
|
||||||
|
def set_up_substitutions(self, tag):
|
||||||
|
"""Replace the declared encoding in a <meta> tag with a placeholder,
|
||||||
|
to be substituted when the tag is output to a string.
|
||||||
|
|
||||||
|
An HTML document may come in to Beautiful Soup as one
|
||||||
|
encoding, but exit in a different encoding, and the <meta> tag
|
||||||
|
needs to be changed to reflect this.
|
||||||
|
|
||||||
|
:param tag: A `Tag`
|
||||||
|
:return: Whether or not a substitution was performed.
|
||||||
|
"""
|
||||||
|
# We are only interested in <meta> tags
|
||||||
|
if tag.name != 'meta':
|
||||||
|
return False
|
||||||
|
|
||||||
|
http_equiv = tag.get('http-equiv')
|
||||||
|
content = tag.get('content')
|
||||||
|
charset = tag.get('charset')
|
||||||
|
|
||||||
|
# We are interested in <meta> tags that say what encoding the
|
||||||
|
# document was originally in. This means HTML 5-style <meta>
|
||||||
|
# tags that provide the "charset" attribute. It also means
|
||||||
|
# HTML 4-style <meta> tags that provide the "content"
|
||||||
|
# attribute and have "http-equiv" set to "content-type".
|
||||||
|
#
|
||||||
|
# In both cases we will replace the value of the appropriate
|
||||||
|
# attribute with a standin object that can take on any
|
||||||
|
# encoding.
|
||||||
|
meta_encoding = None
|
||||||
|
if charset is not None:
|
||||||
|
# HTML 5 style:
|
||||||
|
# <meta charset="utf8">
|
||||||
|
meta_encoding = charset
|
||||||
|
tag['charset'] = CharsetMetaAttributeValue(charset)
|
||||||
|
|
||||||
|
elif (content is not None and http_equiv is not None
|
||||||
|
and http_equiv.lower() == 'content-type'):
|
||||||
|
# HTML 4 style:
|
||||||
|
# <meta http-equiv="content-type" content="text/html; charset=utf8">
|
||||||
|
tag['content'] = ContentMetaAttributeValue(content)
|
||||||
|
|
||||||
|
return (meta_encoding is not None)
|
||||||
|
|
||||||
|
class DetectsXMLParsedAsHTML(object):
|
||||||
|
"""A mixin class for any class (a TreeBuilder, or some class used by a
|
||||||
|
TreeBuilder) that's in a position to detect whether an XML
|
||||||
|
document is being incorrectly parsed as HTML, and issue an
|
||||||
|
appropriate warning.
|
||||||
|
|
||||||
|
This requires being able to observe an incoming processing
|
||||||
|
instruction that might be an XML declaration, and also able to
|
||||||
|
observe tags as they're opened. If you can't do that for a given
|
||||||
|
TreeBuilder, there's a less reliable implementation based on
|
||||||
|
examining the raw markup.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Regular expression for seeing if markup has an <html> tag.
|
||||||
|
LOOKS_LIKE_HTML = re.compile("<[^ +]html", re.I)
|
||||||
|
LOOKS_LIKE_HTML_B = re.compile(b"<[^ +]html", re.I)
|
||||||
|
|
||||||
|
XML_PREFIX = '<?xml'
|
||||||
|
XML_PREFIX_B = b'<?xml'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def warn_if_markup_looks_like_xml(cls, markup):
|
||||||
|
"""Perform a check on some markup to see if it looks like XML
|
||||||
|
that's not XHTML. If so, issue a warning.
|
||||||
|
|
||||||
|
This is much less reliable than doing the check while parsing,
|
||||||
|
but some of the tree builders can't do that.
|
||||||
|
|
||||||
|
:return: True if the markup looks like non-XHTML XML, False
|
||||||
|
otherwise.
|
||||||
|
"""
|
||||||
|
if isinstance(markup, bytes):
|
||||||
|
prefix = cls.XML_PREFIX_B
|
||||||
|
looks_like_html = cls.LOOKS_LIKE_HTML_B
|
||||||
|
else:
|
||||||
|
prefix = cls.XML_PREFIX
|
||||||
|
looks_like_html = cls.LOOKS_LIKE_HTML
|
||||||
|
|
||||||
|
if (markup is not None
|
||||||
|
and markup.startswith(prefix)
|
||||||
|
and not looks_like_html.search(markup[:500])
|
||||||
|
):
|
||||||
|
cls._warn()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _warn(cls):
|
||||||
|
"""Issue a warning about XML being parsed as HTML."""
|
||||||
|
warnings.warn(
|
||||||
|
XMLParsedAsHTMLWarning.MESSAGE, XMLParsedAsHTMLWarning
|
||||||
|
)
|
||||||
|
|
||||||
|
def _initialize_xml_detector(self):
|
||||||
|
"""Call this method before parsing a document."""
|
||||||
|
self._first_processing_instruction = None
|
||||||
|
self._root_tag = None
|
||||||
|
|
||||||
|
def _document_might_be_xml(self, processing_instruction):
|
||||||
|
"""Call this method when encountering an XML declaration, or a
|
||||||
|
"processing instruction" that might be an XML declaration.
|
||||||
|
"""
|
||||||
|
if (self._first_processing_instruction is not None
|
||||||
|
or self._root_tag is not None):
|
||||||
|
# The document has already started. Don't bother checking
|
||||||
|
# anymore.
|
||||||
|
return
|
||||||
|
|
||||||
|
self._first_processing_instruction = processing_instruction
|
||||||
|
|
||||||
|
# We won't know until we encounter the first tag whether or
|
||||||
|
# not this is actually a problem.
|
||||||
|
|
||||||
|
def _root_tag_encountered(self, name):
|
||||||
|
"""Call this when you encounter the document's root tag.
|
||||||
|
|
||||||
|
This is where we actually check whether an XML document is
|
||||||
|
being incorrectly parsed as HTML, and issue the warning.
|
||||||
|
"""
|
||||||
|
if self._root_tag is not None:
|
||||||
|
# This method was incorrectly called multiple times. Do
|
||||||
|
# nothing.
|
||||||
|
return
|
||||||
|
|
||||||
|
self._root_tag = name
|
||||||
|
if (name != 'html' and self._first_processing_instruction is not None
|
||||||
|
and self._first_processing_instruction.lower().startswith('xml ')):
|
||||||
|
# We encountered an XML declaration and then a tag other
|
||||||
|
# than 'html'. This is a reliable indicator that a
|
||||||
|
# non-XHTML document is being parsed as XML.
|
||||||
|
self._warn()
|
||||||
|
|
||||||
|
|
||||||
|
def register_treebuilders_from(module):
|
||||||
|
"""Copy TreeBuilders from the given module into this module."""
|
||||||
|
this_module = sys.modules[__name__]
|
||||||
|
for name in module.__all__:
|
||||||
|
obj = getattr(module, name)
|
||||||
|
|
||||||
|
if issubclass(obj, TreeBuilder):
|
||||||
|
setattr(this_module, name, obj)
|
||||||
|
this_module.__all__.append(name)
|
||||||
|
# Register the builder while we're at it.
|
||||||
|
this_module.builder_registry.register(obj)
|
||||||
|
|
||||||
|
class ParserRejectedMarkup(Exception):
|
||||||
|
"""An Exception to be raised when the underlying parser simply
|
||||||
|
refuses to parse the given markup.
|
||||||
|
"""
|
||||||
|
def __init__(self, message_or_exception):
|
||||||
|
"""Explain why the parser rejected the given markup, either
|
||||||
|
with a textual explanation or another exception.
|
||||||
|
"""
|
||||||
|
if isinstance(message_or_exception, Exception):
|
||||||
|
e = message_or_exception
|
||||||
|
message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
|
||||||
|
super(ParserRejectedMarkup, self).__init__(message_or_exception)
|
||||||
|
|
||||||
|
# Builders are registered in reverse order of priority, so that custom
|
||||||
|
# builder registrations will take precedence. In general, we want lxml
|
||||||
|
# to take precedence over html5lib, because it's faster. And we only
|
||||||
|
# want to use HTMLParser as a last resort.
|
||||||
|
from . import _htmlparser
|
||||||
|
register_treebuilders_from(_htmlparser)
|
||||||
|
try:
|
||||||
|
from . import _html5lib
|
||||||
|
register_treebuilders_from(_html5lib)
|
||||||
|
except ImportError:
|
||||||
|
# They don't have html5lib installed.
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
from . import _lxml
|
||||||
|
register_treebuilders_from(_lxml)
|
||||||
|
except ImportError:
|
||||||
|
# They don't have lxml installed.
|
||||||
|
pass
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
473
acer-env/lib/python3.10/site-packages/bs4/builder/_html5lib.py
Normal file
473
acer-env/lib/python3.10/site-packages/bs4/builder/_html5lib.py
Normal file
@ -0,0 +1,473 @@
|
|||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'HTML5TreeBuilder',
|
||||||
|
]
|
||||||
|
|
||||||
|
import warnings
|
||||||
|
import re
|
||||||
|
from bs4.builder import (
|
||||||
|
DetectsXMLParsedAsHTML,
|
||||||
|
PERMISSIVE,
|
||||||
|
HTML,
|
||||||
|
HTML_5,
|
||||||
|
HTMLTreeBuilder,
|
||||||
|
)
|
||||||
|
from bs4.element import (
|
||||||
|
NamespacedAttribute,
|
||||||
|
nonwhitespace_re,
|
||||||
|
)
|
||||||
|
import html5lib
|
||||||
|
from html5lib.constants import (
|
||||||
|
namespaces,
|
||||||
|
prefixes,
|
||||||
|
)
|
||||||
|
from bs4.element import (
|
||||||
|
Comment,
|
||||||
|
Doctype,
|
||||||
|
NavigableString,
|
||||||
|
Tag,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Pre-0.99999999
|
||||||
|
from html5lib.treebuilders import _base as treebuilder_base
|
||||||
|
new_html5lib = False
|
||||||
|
except ImportError as e:
|
||||||
|
# 0.99999999 and up
|
||||||
|
from html5lib.treebuilders import base as treebuilder_base
|
||||||
|
new_html5lib = True
|
||||||
|
|
||||||
|
class HTML5TreeBuilder(HTMLTreeBuilder):
|
||||||
|
"""Use html5lib to build a tree.
|
||||||
|
|
||||||
|
Note that this TreeBuilder does not support some features common
|
||||||
|
to HTML TreeBuilders. Some of these features could theoretically
|
||||||
|
be implemented, but at the very least it's quite difficult,
|
||||||
|
because html5lib moves the parse tree around as it's being built.
|
||||||
|
|
||||||
|
* This TreeBuilder doesn't use different subclasses of NavigableString
|
||||||
|
based on the name of the tag in which the string was found.
|
||||||
|
|
||||||
|
* You can't use a SoupStrainer to parse only part of a document.
|
||||||
|
"""
|
||||||
|
|
||||||
|
NAME = "html5lib"
|
||||||
|
|
||||||
|
features = [NAME, PERMISSIVE, HTML_5, HTML]
|
||||||
|
|
||||||
|
# html5lib can tell us which line number and position in the
|
||||||
|
# original file is the source of an element.
|
||||||
|
TRACKS_LINE_NUMBERS = True
|
||||||
|
|
||||||
|
def prepare_markup(self, markup, user_specified_encoding,
|
||||||
|
document_declared_encoding=None, exclude_encodings=None):
|
||||||
|
# Store the user-specified encoding for use later on.
|
||||||
|
self.user_specified_encoding = user_specified_encoding
|
||||||
|
|
||||||
|
# document_declared_encoding and exclude_encodings aren't used
|
||||||
|
# ATM because the html5lib TreeBuilder doesn't use
|
||||||
|
# UnicodeDammit.
|
||||||
|
if exclude_encodings:
|
||||||
|
warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
|
||||||
|
|
||||||
|
# html5lib only parses HTML, so if it's given XML that's worth
|
||||||
|
# noting.
|
||||||
|
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
|
||||||
|
|
||||||
|
yield (markup, None, None, False)
|
||||||
|
|
||||||
|
# These methods are defined by Beautiful Soup.
|
||||||
|
def feed(self, markup):
|
||||||
|
if self.soup.parse_only is not None:
|
||||||
|
warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
|
||||||
|
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
|
||||||
|
self.underlying_builder.parser = parser
|
||||||
|
extra_kwargs = dict()
|
||||||
|
if not isinstance(markup, str):
|
||||||
|
if new_html5lib:
|
||||||
|
extra_kwargs['override_encoding'] = self.user_specified_encoding
|
||||||
|
else:
|
||||||
|
extra_kwargs['encoding'] = self.user_specified_encoding
|
||||||
|
doc = parser.parse(markup, **extra_kwargs)
|
||||||
|
|
||||||
|
# Set the character encoding detected by the tokenizer.
|
||||||
|
if isinstance(markup, str):
|
||||||
|
# We need to special-case this because html5lib sets
|
||||||
|
# charEncoding to UTF-8 if it gets Unicode input.
|
||||||
|
doc.original_encoding = None
|
||||||
|
else:
|
||||||
|
original_encoding = parser.tokenizer.stream.charEncoding[0]
|
||||||
|
if not isinstance(original_encoding, str):
|
||||||
|
# In 0.99999999 and up, the encoding is an html5lib
|
||||||
|
# Encoding object. We want to use a string for compatibility
|
||||||
|
# with other tree builders.
|
||||||
|
original_encoding = original_encoding.name
|
||||||
|
doc.original_encoding = original_encoding
|
||||||
|
self.underlying_builder.parser = None
|
||||||
|
|
||||||
|
def create_treebuilder(self, namespaceHTMLElements):
|
||||||
|
self.underlying_builder = TreeBuilderForHtml5lib(
|
||||||
|
namespaceHTMLElements, self.soup,
|
||||||
|
store_line_numbers=self.store_line_numbers
|
||||||
|
)
|
||||||
|
return self.underlying_builder
|
||||||
|
|
||||||
|
def test_fragment_to_document(self, fragment):
|
||||||
|
"""See `TreeBuilder`."""
|
||||||
|
return '<html><head></head><body>%s</body></html>' % fragment
|
||||||
|
|
||||||
|
|
||||||
|
class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
|
||||||
|
|
||||||
|
def __init__(self, namespaceHTMLElements, soup=None,
|
||||||
|
store_line_numbers=True, **kwargs):
|
||||||
|
if soup:
|
||||||
|
self.soup = soup
|
||||||
|
else:
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
# TODO: Why is the parser 'html.parser' here? To avoid an
|
||||||
|
# infinite loop?
|
||||||
|
self.soup = BeautifulSoup(
|
||||||
|
"", "html.parser", store_line_numbers=store_line_numbers,
|
||||||
|
**kwargs
|
||||||
|
)
|
||||||
|
# TODO: What are **kwargs exactly? Should they be passed in
|
||||||
|
# here in addition to/instead of being passed to the BeautifulSoup
|
||||||
|
# constructor?
|
||||||
|
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
|
||||||
|
|
||||||
|
# This will be set later to an html5lib.html5parser.HTMLParser
|
||||||
|
# object, which we can use to track the current line number.
|
||||||
|
self.parser = None
|
||||||
|
self.store_line_numbers = store_line_numbers
|
||||||
|
|
||||||
|
def documentClass(self):
|
||||||
|
self.soup.reset()
|
||||||
|
return Element(self.soup, self.soup, None)
|
||||||
|
|
||||||
|
def insertDoctype(self, token):
|
||||||
|
name = token["name"]
|
||||||
|
publicId = token["publicId"]
|
||||||
|
systemId = token["systemId"]
|
||||||
|
|
||||||
|
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
|
||||||
|
self.soup.object_was_parsed(doctype)
|
||||||
|
|
||||||
|
def elementClass(self, name, namespace):
|
||||||
|
kwargs = {}
|
||||||
|
if self.parser and self.store_line_numbers:
|
||||||
|
# This represents the point immediately after the end of the
|
||||||
|
# tag. We don't know when the tag started, but we do know
|
||||||
|
# where it ended -- the character just before this one.
|
||||||
|
sourceline, sourcepos = self.parser.tokenizer.stream.position()
|
||||||
|
kwargs['sourceline'] = sourceline
|
||||||
|
kwargs['sourcepos'] = sourcepos-1
|
||||||
|
tag = self.soup.new_tag(name, namespace, **kwargs)
|
||||||
|
|
||||||
|
return Element(tag, self.soup, namespace)
|
||||||
|
|
||||||
|
def commentClass(self, data):
|
||||||
|
return TextNode(Comment(data), self.soup)
|
||||||
|
|
||||||
|
def fragmentClass(self):
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
# TODO: Why is the parser 'html.parser' here? To avoid an
|
||||||
|
# infinite loop?
|
||||||
|
self.soup = BeautifulSoup("", "html.parser")
|
||||||
|
self.soup.name = "[document_fragment]"
|
||||||
|
return Element(self.soup, self.soup, None)
|
||||||
|
|
||||||
|
def appendChild(self, node):
|
||||||
|
# XXX This code is not covered by the BS4 tests.
|
||||||
|
self.soup.append(node.element)
|
||||||
|
|
||||||
|
def getDocument(self):
|
||||||
|
return self.soup
|
||||||
|
|
||||||
|
def getFragment(self):
|
||||||
|
return treebuilder_base.TreeBuilder.getFragment(self).element
|
||||||
|
|
||||||
|
def testSerializer(self, element):
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
rv = []
|
||||||
|
doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
|
||||||
|
|
||||||
|
def serializeElement(element, indent=0):
|
||||||
|
if isinstance(element, BeautifulSoup):
|
||||||
|
pass
|
||||||
|
if isinstance(element, Doctype):
|
||||||
|
m = doctype_re.match(element)
|
||||||
|
if m:
|
||||||
|
name = m.group(1)
|
||||||
|
if m.lastindex > 1:
|
||||||
|
publicId = m.group(2) or ""
|
||||||
|
systemId = m.group(3) or m.group(4) or ""
|
||||||
|
rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
|
||||||
|
(' ' * indent, name, publicId, systemId))
|
||||||
|
else:
|
||||||
|
rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
|
||||||
|
else:
|
||||||
|
rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
|
||||||
|
elif isinstance(element, Comment):
|
||||||
|
rv.append("|%s<!-- %s -->" % (' ' * indent, element))
|
||||||
|
elif isinstance(element, NavigableString):
|
||||||
|
rv.append("|%s\"%s\"" % (' ' * indent, element))
|
||||||
|
else:
|
||||||
|
if element.namespace:
|
||||||
|
name = "%s %s" % (prefixes[element.namespace],
|
||||||
|
element.name)
|
||||||
|
else:
|
||||||
|
name = element.name
|
||||||
|
rv.append("|%s<%s>" % (' ' * indent, name))
|
||||||
|
if element.attrs:
|
||||||
|
attributes = []
|
||||||
|
for name, value in list(element.attrs.items()):
|
||||||
|
if isinstance(name, NamespacedAttribute):
|
||||||
|
name = "%s %s" % (prefixes[name.namespace], name.name)
|
||||||
|
if isinstance(value, list):
|
||||||
|
value = " ".join(value)
|
||||||
|
attributes.append((name, value))
|
||||||
|
|
||||||
|
for name, value in sorted(attributes):
|
||||||
|
rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
|
||||||
|
indent += 2
|
||||||
|
for child in element.children:
|
||||||
|
serializeElement(child, indent)
|
||||||
|
serializeElement(element, 0)
|
||||||
|
|
||||||
|
return "\n".join(rv)
|
||||||
|
|
||||||
|
class AttrList(object):
|
||||||
|
def __init__(self, element):
|
||||||
|
self.element = element
|
||||||
|
self.attrs = dict(self.element.attrs)
|
||||||
|
def __iter__(self):
|
||||||
|
return list(self.attrs.items()).__iter__()
|
||||||
|
def __setitem__(self, name, value):
|
||||||
|
# If this attribute is a multi-valued attribute for this element,
|
||||||
|
# turn its value into a list.
|
||||||
|
list_attr = self.element.cdata_list_attributes or {}
|
||||||
|
if (name in list_attr.get('*')
|
||||||
|
or (self.element.name in list_attr
|
||||||
|
and name in list_attr[self.element.name])):
|
||||||
|
# A node that is being cloned may have already undergone
|
||||||
|
# this procedure.
|
||||||
|
if not isinstance(value, list):
|
||||||
|
value = nonwhitespace_re.findall(value)
|
||||||
|
self.element[name] = value
|
||||||
|
def items(self):
|
||||||
|
return list(self.attrs.items())
|
||||||
|
def keys(self):
|
||||||
|
return list(self.attrs.keys())
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.attrs)
|
||||||
|
def __getitem__(self, name):
|
||||||
|
return self.attrs[name]
|
||||||
|
def __contains__(self, name):
|
||||||
|
return name in list(self.attrs.keys())
|
||||||
|
|
||||||
|
|
||||||
|
class Element(treebuilder_base.Node):
|
||||||
|
def __init__(self, element, soup, namespace):
|
||||||
|
treebuilder_base.Node.__init__(self, element.name)
|
||||||
|
self.element = element
|
||||||
|
self.soup = soup
|
||||||
|
self.namespace = namespace
|
||||||
|
|
||||||
|
def appendChild(self, node):
|
||||||
|
string_child = child = None
|
||||||
|
if isinstance(node, str):
|
||||||
|
# Some other piece of code decided to pass in a string
|
||||||
|
# instead of creating a TextElement object to contain the
|
||||||
|
# string.
|
||||||
|
string_child = child = node
|
||||||
|
elif isinstance(node, Tag):
|
||||||
|
# Some other piece of code decided to pass in a Tag
|
||||||
|
# instead of creating an Element object to contain the
|
||||||
|
# Tag.
|
||||||
|
child = node
|
||||||
|
elif node.element.__class__ == NavigableString:
|
||||||
|
string_child = child = node.element
|
||||||
|
node.parent = self
|
||||||
|
else:
|
||||||
|
child = node.element
|
||||||
|
node.parent = self
|
||||||
|
|
||||||
|
if not isinstance(child, str) and child.parent is not None:
|
||||||
|
node.element.extract()
|
||||||
|
|
||||||
|
if (string_child is not None and self.element.contents
|
||||||
|
and self.element.contents[-1].__class__ == NavigableString):
|
||||||
|
# We are appending a string onto another string.
|
||||||
|
# TODO This has O(n^2) performance, for input like
|
||||||
|
# "a</a>a</a>a</a>..."
|
||||||
|
old_element = self.element.contents[-1]
|
||||||
|
new_element = self.soup.new_string(old_element + string_child)
|
||||||
|
old_element.replace_with(new_element)
|
||||||
|
self.soup._most_recent_element = new_element
|
||||||
|
else:
|
||||||
|
if isinstance(node, str):
|
||||||
|
# Create a brand new NavigableString from this string.
|
||||||
|
child = self.soup.new_string(node)
|
||||||
|
|
||||||
|
# Tell Beautiful Soup to act as if it parsed this element
|
||||||
|
# immediately after the parent's last descendant. (Or
|
||||||
|
# immediately after the parent, if it has no children.)
|
||||||
|
if self.element.contents:
|
||||||
|
most_recent_element = self.element._last_descendant(False)
|
||||||
|
elif self.element.next_element is not None:
|
||||||
|
# Something from further ahead in the parse tree is
|
||||||
|
# being inserted into this earlier element. This is
|
||||||
|
# very annoying because it means an expensive search
|
||||||
|
# for the last element in the tree.
|
||||||
|
most_recent_element = self.soup._last_descendant()
|
||||||
|
else:
|
||||||
|
most_recent_element = self.element
|
||||||
|
|
||||||
|
self.soup.object_was_parsed(
|
||||||
|
child, parent=self.element,
|
||||||
|
most_recent_element=most_recent_element)
|
||||||
|
|
||||||
|
def getAttributes(self):
|
||||||
|
if isinstance(self.element, Comment):
|
||||||
|
return {}
|
||||||
|
return AttrList(self.element)
|
||||||
|
|
||||||
|
def setAttributes(self, attributes):
|
||||||
|
if attributes is not None and len(attributes) > 0:
|
||||||
|
converted_attributes = []
|
||||||
|
for name, value in list(attributes.items()):
|
||||||
|
if isinstance(name, tuple):
|
||||||
|
new_name = NamespacedAttribute(*name)
|
||||||
|
del attributes[name]
|
||||||
|
attributes[new_name] = value
|
||||||
|
|
||||||
|
self.soup.builder._replace_cdata_list_attribute_values(
|
||||||
|
self.name, attributes)
|
||||||
|
for name, value in list(attributes.items()):
|
||||||
|
self.element[name] = value
|
||||||
|
|
||||||
|
# The attributes may contain variables that need substitution.
|
||||||
|
# Call set_up_substitutions manually.
|
||||||
|
#
|
||||||
|
# The Tag constructor called this method when the Tag was created,
|
||||||
|
# but we just set/changed the attributes, so call it again.
|
||||||
|
self.soup.builder.set_up_substitutions(self.element)
|
||||||
|
attributes = property(getAttributes, setAttributes)
|
||||||
|
|
||||||
|
def insertText(self, data, insertBefore=None):
|
||||||
|
text = TextNode(self.soup.new_string(data), self.soup)
|
||||||
|
if insertBefore:
|
||||||
|
self.insertBefore(text, insertBefore)
|
||||||
|
else:
|
||||||
|
self.appendChild(text)
|
||||||
|
|
||||||
|
def insertBefore(self, node, refNode):
|
||||||
|
index = self.element.index(refNode.element)
|
||||||
|
if (node.element.__class__ == NavigableString and self.element.contents
|
||||||
|
and self.element.contents[index-1].__class__ == NavigableString):
|
||||||
|
# (See comments in appendChild)
|
||||||
|
old_node = self.element.contents[index-1]
|
||||||
|
new_str = self.soup.new_string(old_node + node.element)
|
||||||
|
old_node.replace_with(new_str)
|
||||||
|
else:
|
||||||
|
self.element.insert(index, node.element)
|
||||||
|
node.parent = self
|
||||||
|
|
||||||
|
def removeChild(self, node):
|
||||||
|
node.element.extract()
|
||||||
|
|
||||||
|
def reparentChildren(self, new_parent):
|
||||||
|
"""Move all of this tag's children into another tag."""
|
||||||
|
# print("MOVE", self.element.contents)
|
||||||
|
# print("FROM", self.element)
|
||||||
|
# print("TO", new_parent.element)
|
||||||
|
|
||||||
|
element = self.element
|
||||||
|
new_parent_element = new_parent.element
|
||||||
|
# Determine what this tag's next_element will be once all the children
|
||||||
|
# are removed.
|
||||||
|
final_next_element = element.next_sibling
|
||||||
|
|
||||||
|
new_parents_last_descendant = new_parent_element._last_descendant(False, False)
|
||||||
|
if len(new_parent_element.contents) > 0:
|
||||||
|
# The new parent already contains children. We will be
|
||||||
|
# appending this tag's children to the end.
|
||||||
|
new_parents_last_child = new_parent_element.contents[-1]
|
||||||
|
new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
|
||||||
|
else:
|
||||||
|
# The new parent contains no children.
|
||||||
|
new_parents_last_child = None
|
||||||
|
new_parents_last_descendant_next_element = new_parent_element.next_element
|
||||||
|
|
||||||
|
to_append = element.contents
|
||||||
|
if len(to_append) > 0:
|
||||||
|
# Set the first child's previous_element and previous_sibling
|
||||||
|
# to elements within the new parent
|
||||||
|
first_child = to_append[0]
|
||||||
|
if new_parents_last_descendant is not None:
|
||||||
|
first_child.previous_element = new_parents_last_descendant
|
||||||
|
else:
|
||||||
|
first_child.previous_element = new_parent_element
|
||||||
|
first_child.previous_sibling = new_parents_last_child
|
||||||
|
if new_parents_last_descendant is not None:
|
||||||
|
new_parents_last_descendant.next_element = first_child
|
||||||
|
else:
|
||||||
|
new_parent_element.next_element = first_child
|
||||||
|
if new_parents_last_child is not None:
|
||||||
|
new_parents_last_child.next_sibling = first_child
|
||||||
|
|
||||||
|
# Find the very last element being moved. It is now the
|
||||||
|
# parent's last descendant. It has no .next_sibling and
|
||||||
|
# its .next_element is whatever the previous last
|
||||||
|
# descendant had.
|
||||||
|
last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
|
||||||
|
|
||||||
|
last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
|
||||||
|
if new_parents_last_descendant_next_element is not None:
|
||||||
|
# TODO: This code has no test coverage and I'm not sure
|
||||||
|
# how to get html5lib to go through this path, but it's
|
||||||
|
# just the other side of the previous line.
|
||||||
|
new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
|
||||||
|
last_childs_last_descendant.next_sibling = None
|
||||||
|
|
||||||
|
for child in to_append:
|
||||||
|
child.parent = new_parent_element
|
||||||
|
new_parent_element.contents.append(child)
|
||||||
|
|
||||||
|
# Now that this element has no children, change its .next_element.
|
||||||
|
element.contents = []
|
||||||
|
element.next_element = final_next_element
|
||||||
|
|
||||||
|
# print("DONE WITH MOVE")
|
||||||
|
# print("FROM", self.element)
|
||||||
|
# print("TO", new_parent_element)
|
||||||
|
|
||||||
|
def cloneNode(self):
|
||||||
|
tag = self.soup.new_tag(self.element.name, self.namespace)
|
||||||
|
node = Element(tag, self.soup, self.namespace)
|
||||||
|
for key,value in self.attributes:
|
||||||
|
node.attributes[key] = value
|
||||||
|
return node
|
||||||
|
|
||||||
|
def hasContent(self):
|
||||||
|
return self.element.contents
|
||||||
|
|
||||||
|
def getNameTuple(self):
|
||||||
|
if self.namespace == None:
|
||||||
|
return namespaces["html"], self.name
|
||||||
|
else:
|
||||||
|
return self.namespace, self.name
|
||||||
|
|
||||||
|
nameTuple = property(getNameTuple)
|
||||||
|
|
||||||
|
class TextNode(Element):
|
||||||
|
def __init__(self, element, soup):
|
||||||
|
treebuilder_base.Node.__init__(self, None)
|
||||||
|
self.element = element
|
||||||
|
self.soup = soup
|
||||||
|
|
||||||
|
def cloneNode(self):
|
||||||
|
raise NotImplementedError
|
499
acer-env/lib/python3.10/site-packages/bs4/builder/_htmlparser.py
Normal file
499
acer-env/lib/python3.10/site-packages/bs4/builder/_htmlparser.py
Normal file
@ -0,0 +1,499 @@
|
|||||||
|
# encoding: utf-8
|
||||||
|
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
|
||||||
|
|
||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'HTMLParserTreeBuilder',
|
||||||
|
]
|
||||||
|
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
|
||||||
|
try:
|
||||||
|
from html.parser import HTMLParseError
|
||||||
|
except ImportError as e:
|
||||||
|
# HTMLParseError is removed in Python 3.5. Since it can never be
|
||||||
|
# thrown in 3.5, we can just define our own class as a placeholder.
|
||||||
|
class HTMLParseError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
|
||||||
|
# argument, which we'd like to set to False. Unfortunately,
|
||||||
|
# http://bugs.python.org/issue13273 makes strict=True a better bet
|
||||||
|
# before Python 3.2.3.
|
||||||
|
#
|
||||||
|
# At the end of this file, we monkeypatch HTMLParser so that
|
||||||
|
# strict=True works well on Python 3.2.2.
|
||||||
|
major, minor, release = sys.version_info[:3]
|
||||||
|
CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
|
||||||
|
CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
|
||||||
|
CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
|
||||||
|
|
||||||
|
|
||||||
|
from bs4.element import (
|
||||||
|
CData,
|
||||||
|
Comment,
|
||||||
|
Declaration,
|
||||||
|
Doctype,
|
||||||
|
ProcessingInstruction,
|
||||||
|
)
|
||||||
|
from bs4.dammit import EntitySubstitution, UnicodeDammit
|
||||||
|
|
||||||
|
from bs4.builder import (
|
||||||
|
DetectsXMLParsedAsHTML,
|
||||||
|
HTML,
|
||||||
|
HTMLTreeBuilder,
|
||||||
|
STRICT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
HTMLPARSER = 'html.parser'
|
||||||
|
|
||||||
|
class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
|
||||||
|
"""A subclass of the Python standard library's HTMLParser class, which
|
||||||
|
listens for HTMLParser events and translates them into calls
|
||||||
|
to Beautiful Soup's tree construction API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Strategies for handling duplicate attributes
|
||||||
|
IGNORE = 'ignore'
|
||||||
|
REPLACE = 'replace'
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param on_duplicate_attribute: A strategy for what to do if a
|
||||||
|
tag includes the same attribute more than once. Accepted
|
||||||
|
values are: REPLACE (replace earlier values with later
|
||||||
|
ones, the default), IGNORE (keep the earliest value
|
||||||
|
encountered), or a callable. A callable must take three
|
||||||
|
arguments: the dictionary of attributes already processed,
|
||||||
|
the name of the duplicate attribute, and the most recent value
|
||||||
|
encountered.
|
||||||
|
"""
|
||||||
|
self.on_duplicate_attribute = kwargs.pop(
|
||||||
|
'on_duplicate_attribute', self.REPLACE
|
||||||
|
)
|
||||||
|
HTMLParser.__init__(self, *args, **kwargs)
|
||||||
|
|
||||||
|
# Keep a list of empty-element tags that were encountered
|
||||||
|
# without an explicit closing tag. If we encounter a closing tag
|
||||||
|
# of this type, we'll associate it with one of those entries.
|
||||||
|
#
|
||||||
|
# This isn't a stack because we don't care about the
|
||||||
|
# order. It's a list of closing tags we've already handled and
|
||||||
|
# will ignore, assuming they ever show up.
|
||||||
|
self.already_closed_empty_element = []
|
||||||
|
|
||||||
|
self._initialize_xml_detector()
|
||||||
|
|
||||||
|
def error(self, msg):
|
||||||
|
"""In Python 3, HTMLParser subclasses must implement error(), although
|
||||||
|
this requirement doesn't appear to be documented.
|
||||||
|
|
||||||
|
In Python 2, HTMLParser implements error() by raising an exception,
|
||||||
|
which we don't want to do.
|
||||||
|
|
||||||
|
In any event, this method is called only on very strange
|
||||||
|
markup and our best strategy is to pretend it didn't happen
|
||||||
|
and keep going.
|
||||||
|
"""
|
||||||
|
warnings.warn(msg)
|
||||||
|
|
||||||
|
def handle_startendtag(self, name, attrs):
|
||||||
|
"""Handle an incoming empty-element tag.
|
||||||
|
|
||||||
|
This is only called when the markup looks like <tag/>.
|
||||||
|
|
||||||
|
:param name: Name of the tag.
|
||||||
|
:param attrs: Dictionary of the tag's attributes.
|
||||||
|
"""
|
||||||
|
# is_startend() tells handle_starttag not to close the tag
|
||||||
|
# just because its name matches a known empty-element tag. We
|
||||||
|
# know that this is an empty-element tag and we want to call
|
||||||
|
# handle_endtag ourselves.
|
||||||
|
tag = self.handle_starttag(name, attrs, handle_empty_element=False)
|
||||||
|
self.handle_endtag(name)
|
||||||
|
|
||||||
|
def handle_starttag(self, name, attrs, handle_empty_element=True):
|
||||||
|
"""Handle an opening tag, e.g. '<tag>'
|
||||||
|
|
||||||
|
:param name: Name of the tag.
|
||||||
|
:param attrs: Dictionary of the tag's attributes.
|
||||||
|
:param handle_empty_element: True if this tag is known to be
|
||||||
|
an empty-element tag (i.e. there is not expected to be any
|
||||||
|
closing tag).
|
||||||
|
"""
|
||||||
|
# XXX namespace
|
||||||
|
attr_dict = {}
|
||||||
|
for key, value in attrs:
|
||||||
|
# Change None attribute values to the empty string
|
||||||
|
# for consistency with the other tree builders.
|
||||||
|
if value is None:
|
||||||
|
value = ''
|
||||||
|
if key in attr_dict:
|
||||||
|
# A single attribute shows up multiple times in this
|
||||||
|
# tag. How to handle it depends on the
|
||||||
|
# on_duplicate_attribute setting.
|
||||||
|
on_dupe = self.on_duplicate_attribute
|
||||||
|
if on_dupe == self.IGNORE:
|
||||||
|
pass
|
||||||
|
elif on_dupe in (None, self.REPLACE):
|
||||||
|
attr_dict[key] = value
|
||||||
|
else:
|
||||||
|
on_dupe(attr_dict, key, value)
|
||||||
|
else:
|
||||||
|
attr_dict[key] = value
|
||||||
|
attrvalue = '""'
|
||||||
|
#print("START", name)
|
||||||
|
sourceline, sourcepos = self.getpos()
|
||||||
|
tag = self.soup.handle_starttag(
|
||||||
|
name, None, None, attr_dict, sourceline=sourceline,
|
||||||
|
sourcepos=sourcepos
|
||||||
|
)
|
||||||
|
if tag and tag.is_empty_element and handle_empty_element:
|
||||||
|
# Unlike other parsers, html.parser doesn't send separate end tag
|
||||||
|
# events for empty-element tags. (It's handled in
|
||||||
|
# handle_startendtag, but only if the original markup looked like
|
||||||
|
# <tag/>.)
|
||||||
|
#
|
||||||
|
# So we need to call handle_endtag() ourselves. Since we
|
||||||
|
# know the start event is identical to the end event, we
|
||||||
|
# don't want handle_endtag() to cross off any previous end
|
||||||
|
# events for tags of this name.
|
||||||
|
self.handle_endtag(name, check_already_closed=False)
|
||||||
|
|
||||||
|
# But we might encounter an explicit closing tag for this tag
|
||||||
|
# later on. If so, we want to ignore it.
|
||||||
|
self.already_closed_empty_element.append(name)
|
||||||
|
|
||||||
|
if self._root_tag is None:
|
||||||
|
self._root_tag_encountered(name)
|
||||||
|
|
||||||
|
def handle_endtag(self, name, check_already_closed=True):
|
||||||
|
"""Handle a closing tag, e.g. '</tag>'
|
||||||
|
|
||||||
|
:param name: A tag name.
|
||||||
|
:param check_already_closed: True if this tag is expected to
|
||||||
|
be the closing portion of an empty-element tag,
|
||||||
|
e.g. '<tag></tag>'.
|
||||||
|
"""
|
||||||
|
#print("END", name)
|
||||||
|
if check_already_closed and name in self.already_closed_empty_element:
|
||||||
|
# This is a redundant end tag for an empty-element tag.
|
||||||
|
# We've already called handle_endtag() for it, so just
|
||||||
|
# check it off the list.
|
||||||
|
#print("ALREADY CLOSED", name)
|
||||||
|
self.already_closed_empty_element.remove(name)
|
||||||
|
else:
|
||||||
|
self.soup.handle_endtag(name)
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
"""Handle some textual data that shows up between tags."""
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
|
||||||
|
def handle_charref(self, name):
|
||||||
|
"""Handle a numeric character reference by converting it to the
|
||||||
|
corresponding Unicode character and treating it as textual
|
||||||
|
data.
|
||||||
|
|
||||||
|
:param name: Character number, possibly in hexadecimal.
|
||||||
|
"""
|
||||||
|
# XXX workaround for a bug in HTMLParser. Remove this once
|
||||||
|
# it's fixed in all supported versions.
|
||||||
|
# http://bugs.python.org/issue13633
|
||||||
|
if name.startswith('x'):
|
||||||
|
real_name = int(name.lstrip('x'), 16)
|
||||||
|
elif name.startswith('X'):
|
||||||
|
real_name = int(name.lstrip('X'), 16)
|
||||||
|
else:
|
||||||
|
real_name = int(name)
|
||||||
|
|
||||||
|
data = None
|
||||||
|
if real_name < 256:
|
||||||
|
# HTML numeric entities are supposed to reference Unicode
|
||||||
|
# code points, but sometimes they reference code points in
|
||||||
|
# some other encoding (ahem, Windows-1252). E.g. “
|
||||||
|
# instead of É for LEFT DOUBLE QUOTATION MARK. This
|
||||||
|
# code tries to detect this situation and compensate.
|
||||||
|
for encoding in (self.soup.original_encoding, 'windows-1252'):
|
||||||
|
if not encoding:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = bytearray([real_name]).decode(encoding)
|
||||||
|
except UnicodeDecodeError as e:
|
||||||
|
pass
|
||||||
|
if not data:
|
||||||
|
try:
|
||||||
|
data = chr(real_name)
|
||||||
|
except (ValueError, OverflowError) as e:
|
||||||
|
pass
|
||||||
|
data = data or "\N{REPLACEMENT CHARACTER}"
|
||||||
|
self.handle_data(data)
|
||||||
|
|
||||||
|
def handle_entityref(self, name):
|
||||||
|
"""Handle a named entity reference by converting it to the
|
||||||
|
corresponding Unicode character(s) and treating it as textual
|
||||||
|
data.
|
||||||
|
|
||||||
|
:param name: Name of the entity reference.
|
||||||
|
"""
|
||||||
|
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
|
||||||
|
if character is not None:
|
||||||
|
data = character
|
||||||
|
else:
|
||||||
|
# If this were XML, it would be ambiguous whether "&foo"
|
||||||
|
# was an character entity reference with a missing
|
||||||
|
# semicolon or the literal string "&foo". Since this is
|
||||||
|
# HTML, we have a complete list of all character entity references,
|
||||||
|
# and this one wasn't found, so assume it's the literal string "&foo".
|
||||||
|
data = "&%s" % name
|
||||||
|
self.handle_data(data)
|
||||||
|
|
||||||
|
def handle_comment(self, data):
|
||||||
|
"""Handle an HTML comment.
|
||||||
|
|
||||||
|
:param data: The text of the comment.
|
||||||
|
"""
|
||||||
|
self.soup.endData()
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
self.soup.endData(Comment)
|
||||||
|
|
||||||
|
def handle_decl(self, data):
|
||||||
|
"""Handle a DOCTYPE declaration.
|
||||||
|
|
||||||
|
:param data: The text of the declaration.
|
||||||
|
"""
|
||||||
|
self.soup.endData()
|
||||||
|
data = data[len("DOCTYPE "):]
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
self.soup.endData(Doctype)
|
||||||
|
|
||||||
|
def unknown_decl(self, data):
|
||||||
|
"""Handle a declaration of unknown type -- probably a CDATA block.
|
||||||
|
|
||||||
|
:param data: The text of the declaration.
|
||||||
|
"""
|
||||||
|
if data.upper().startswith('CDATA['):
|
||||||
|
cls = CData
|
||||||
|
data = data[len('CDATA['):]
|
||||||
|
else:
|
||||||
|
cls = Declaration
|
||||||
|
self.soup.endData()
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
self.soup.endData(cls)
|
||||||
|
|
||||||
|
def handle_pi(self, data):
|
||||||
|
"""Handle a processing instruction.
|
||||||
|
|
||||||
|
:param data: The text of the instruction.
|
||||||
|
"""
|
||||||
|
self.soup.endData()
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
self._document_might_be_xml(data)
|
||||||
|
self.soup.endData(ProcessingInstruction)
|
||||||
|
|
||||||
|
|
||||||
|
class HTMLParserTreeBuilder(HTMLTreeBuilder):
|
||||||
|
"""A Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser,
|
||||||
|
found in the Python standard library.
|
||||||
|
"""
|
||||||
|
is_xml = False
|
||||||
|
picklable = True
|
||||||
|
NAME = HTMLPARSER
|
||||||
|
features = [NAME, HTML, STRICT]
|
||||||
|
|
||||||
|
# The html.parser knows which line number and position in the
|
||||||
|
# original file is the source of an element.
|
||||||
|
TRACKS_LINE_NUMBERS = True
|
||||||
|
|
||||||
|
def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param parser_args: Positional arguments to pass into
|
||||||
|
the BeautifulSoupHTMLParser constructor, once it's
|
||||||
|
invoked.
|
||||||
|
:param parser_kwargs: Keyword arguments to pass into
|
||||||
|
the BeautifulSoupHTMLParser constructor, once it's
|
||||||
|
invoked.
|
||||||
|
:param kwargs: Keyword arguments for the superclass constructor.
|
||||||
|
"""
|
||||||
|
# Some keyword arguments will be pulled out of kwargs and placed
|
||||||
|
# into parser_kwargs.
|
||||||
|
extra_parser_kwargs = dict()
|
||||||
|
for arg in ('on_duplicate_attribute',):
|
||||||
|
if arg in kwargs:
|
||||||
|
value = kwargs.pop(arg)
|
||||||
|
extra_parser_kwargs[arg] = value
|
||||||
|
super(HTMLParserTreeBuilder, self).__init__(**kwargs)
|
||||||
|
parser_args = parser_args or []
|
||||||
|
parser_kwargs = parser_kwargs or {}
|
||||||
|
parser_kwargs.update(extra_parser_kwargs)
|
||||||
|
if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
|
||||||
|
parser_kwargs['strict'] = False
|
||||||
|
if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
|
||||||
|
parser_kwargs['convert_charrefs'] = False
|
||||||
|
self.parser_args = (parser_args, parser_kwargs)
|
||||||
|
|
||||||
|
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||||
|
document_declared_encoding=None, exclude_encodings=None):
|
||||||
|
|
||||||
|
"""Run any preliminary steps necessary to make incoming markup
|
||||||
|
acceptable to the parser.
|
||||||
|
|
||||||
|
:param markup: Some markup -- probably a bytestring.
|
||||||
|
:param user_specified_encoding: The user asked to try this encoding.
|
||||||
|
:param document_declared_encoding: The markup itself claims to be
|
||||||
|
in this encoding.
|
||||||
|
:param exclude_encodings: The user asked _not_ to try any of
|
||||||
|
these encodings.
|
||||||
|
|
||||||
|
:yield: A series of 4-tuples:
|
||||||
|
(markup, encoding, declared encoding,
|
||||||
|
has undergone character replacement)
|
||||||
|
|
||||||
|
Each 4-tuple represents a strategy for converting the
|
||||||
|
document to Unicode and parsing it. Each strategy will be tried
|
||||||
|
in turn.
|
||||||
|
"""
|
||||||
|
if isinstance(markup, str):
|
||||||
|
# Parse Unicode as-is.
|
||||||
|
yield (markup, None, None, False)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Ask UnicodeDammit to sniff the most likely encoding.
|
||||||
|
|
||||||
|
# This was provided by the end-user; treat it as a known
|
||||||
|
# definite encoding per the algorithm laid out in the HTML5
|
||||||
|
# spec. (See the EncodingDetector class for details.)
|
||||||
|
known_definite_encodings = [user_specified_encoding]
|
||||||
|
|
||||||
|
# This was found in the document; treat it as a slightly lower-priority
|
||||||
|
# user encoding.
|
||||||
|
user_encodings = [document_declared_encoding]
|
||||||
|
|
||||||
|
try_encodings = [user_specified_encoding, document_declared_encoding]
|
||||||
|
dammit = UnicodeDammit(
|
||||||
|
markup,
|
||||||
|
known_definite_encodings=known_definite_encodings,
|
||||||
|
user_encodings=user_encodings,
|
||||||
|
is_html=True,
|
||||||
|
exclude_encodings=exclude_encodings
|
||||||
|
)
|
||||||
|
yield (dammit.markup, dammit.original_encoding,
|
||||||
|
dammit.declared_html_encoding,
|
||||||
|
dammit.contains_replacement_characters)
|
||||||
|
|
||||||
|
def feed(self, markup):
|
||||||
|
"""Run some incoming markup through some parsing process,
|
||||||
|
populating the `BeautifulSoup` object in self.soup.
|
||||||
|
"""
|
||||||
|
args, kwargs = self.parser_args
|
||||||
|
parser = BeautifulSoupHTMLParser(*args, **kwargs)
|
||||||
|
parser.soup = self.soup
|
||||||
|
try:
|
||||||
|
parser.feed(markup)
|
||||||
|
parser.close()
|
||||||
|
except HTMLParseError as e:
|
||||||
|
warnings.warn(RuntimeWarning(
|
||||||
|
"Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
|
||||||
|
raise e
|
||||||
|
parser.already_closed_empty_element = []
|
||||||
|
|
||||||
|
# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
|
||||||
|
# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
|
||||||
|
# string.
|
||||||
|
#
|
||||||
|
# XXX This code can be removed once most Python 3 users are on 3.2.3.
|
||||||
|
if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
|
||||||
|
import re
|
||||||
|
attrfind_tolerant = re.compile(
|
||||||
|
r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
|
||||||
|
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
|
||||||
|
HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
|
||||||
|
|
||||||
|
locatestarttagend = re.compile(r"""
|
||||||
|
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
|
||||||
|
(?:\s+ # whitespace before attribute name
|
||||||
|
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
|
||||||
|
(?:\s*=\s* # value indicator
|
||||||
|
(?:'[^']*' # LITA-enclosed value
|
||||||
|
|\"[^\"]*\" # LIT-enclosed value
|
||||||
|
|[^'\">\s]+ # bare value
|
||||||
|
)
|
||||||
|
)?
|
||||||
|
)
|
||||||
|
)*
|
||||||
|
\s* # trailing whitespace
|
||||||
|
""", re.VERBOSE)
|
||||||
|
BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
|
||||||
|
|
||||||
|
from html.parser import tagfind, attrfind
|
||||||
|
|
||||||
|
def parse_starttag(self, i):
|
||||||
|
self.__starttag_text = None
|
||||||
|
endpos = self.check_for_whole_start_tag(i)
|
||||||
|
if endpos < 0:
|
||||||
|
return endpos
|
||||||
|
rawdata = self.rawdata
|
||||||
|
self.__starttag_text = rawdata[i:endpos]
|
||||||
|
|
||||||
|
# Now parse the data between i+1 and j into a tag and attrs
|
||||||
|
attrs = []
|
||||||
|
match = tagfind.match(rawdata, i+1)
|
||||||
|
assert match, 'unexpected call to parse_starttag()'
|
||||||
|
k = match.end()
|
||||||
|
self.lasttag = tag = rawdata[i+1:k].lower()
|
||||||
|
while k < endpos:
|
||||||
|
if self.strict:
|
||||||
|
m = attrfind.match(rawdata, k)
|
||||||
|
else:
|
||||||
|
m = attrfind_tolerant.match(rawdata, k)
|
||||||
|
if not m:
|
||||||
|
break
|
||||||
|
attrname, rest, attrvalue = m.group(1, 2, 3)
|
||||||
|
if not rest:
|
||||||
|
attrvalue = None
|
||||||
|
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
|
||||||
|
attrvalue[:1] == '"' == attrvalue[-1:]:
|
||||||
|
attrvalue = attrvalue[1:-1]
|
||||||
|
if attrvalue:
|
||||||
|
attrvalue = self.unescape(attrvalue)
|
||||||
|
attrs.append((attrname.lower(), attrvalue))
|
||||||
|
k = m.end()
|
||||||
|
|
||||||
|
end = rawdata[k:endpos].strip()
|
||||||
|
if end not in (">", "/>"):
|
||||||
|
lineno, offset = self.getpos()
|
||||||
|
if "\n" in self.__starttag_text:
|
||||||
|
lineno = lineno + self.__starttag_text.count("\n")
|
||||||
|
offset = len(self.__starttag_text) \
|
||||||
|
- self.__starttag_text.rfind("\n")
|
||||||
|
else:
|
||||||
|
offset = offset + len(self.__starttag_text)
|
||||||
|
if self.strict:
|
||||||
|
self.error("junk characters in start tag: %r"
|
||||||
|
% (rawdata[k:endpos][:20],))
|
||||||
|
self.handle_data(rawdata[i:endpos])
|
||||||
|
return endpos
|
||||||
|
if end.endswith('/>'):
|
||||||
|
# XHTML-style empty tag: <span attr="value" />
|
||||||
|
self.handle_startendtag(tag, attrs)
|
||||||
|
else:
|
||||||
|
self.handle_starttag(tag, attrs)
|
||||||
|
if tag in self.CDATA_CONTENT_ELEMENTS:
|
||||||
|
self.set_cdata_mode(tag)
|
||||||
|
return endpos
|
||||||
|
|
||||||
|
def set_cdata_mode(self, elem):
|
||||||
|
self.cdata_elem = elem.lower()
|
||||||
|
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
|
||||||
|
|
||||||
|
BeautifulSoupHTMLParser.parse_starttag = parse_starttag
|
||||||
|
BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
|
||||||
|
|
||||||
|
CONSTRUCTOR_TAKES_STRICT = True
|
386
acer-env/lib/python3.10/site-packages/bs4/builder/_lxml.py
Normal file
386
acer-env/lib/python3.10/site-packages/bs4/builder/_lxml.py
Normal file
@ -0,0 +1,386 @@
|
|||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'LXMLTreeBuilderForXML',
|
||||||
|
'LXMLTreeBuilder',
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
from collections.abc import Callable # Python 3.6
|
||||||
|
except ImportError as e:
|
||||||
|
from collections import Callable
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
from io import StringIO
|
||||||
|
from lxml import etree
|
||||||
|
from bs4.element import (
|
||||||
|
Comment,
|
||||||
|
Doctype,
|
||||||
|
NamespacedAttribute,
|
||||||
|
ProcessingInstruction,
|
||||||
|
XMLProcessingInstruction,
|
||||||
|
)
|
||||||
|
from bs4.builder import (
|
||||||
|
DetectsXMLParsedAsHTML,
|
||||||
|
FAST,
|
||||||
|
HTML,
|
||||||
|
HTMLTreeBuilder,
|
||||||
|
PERMISSIVE,
|
||||||
|
ParserRejectedMarkup,
|
||||||
|
TreeBuilder,
|
||||||
|
XML)
|
||||||
|
from bs4.dammit import EncodingDetector
|
||||||
|
|
||||||
|
LXML = 'lxml'
|
||||||
|
|
||||||
|
def _invert(d):
|
||||||
|
"Invert a dictionary."
|
||||||
|
return dict((v,k) for k, v in list(d.items()))
|
||||||
|
|
||||||
|
class LXMLTreeBuilderForXML(TreeBuilder):
|
||||||
|
DEFAULT_PARSER_CLASS = etree.XMLParser
|
||||||
|
|
||||||
|
is_xml = True
|
||||||
|
processing_instruction_class = XMLProcessingInstruction
|
||||||
|
|
||||||
|
NAME = "lxml-xml"
|
||||||
|
ALTERNATE_NAMES = ["xml"]
|
||||||
|
|
||||||
|
# Well, it's permissive by XML parser standards.
|
||||||
|
features = [NAME, LXML, XML, FAST, PERMISSIVE]
|
||||||
|
|
||||||
|
CHUNK_SIZE = 512
|
||||||
|
|
||||||
|
# This namespace mapping is specified in the XML Namespace
|
||||||
|
# standard.
|
||||||
|
DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
|
||||||
|
|
||||||
|
DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
|
||||||
|
|
||||||
|
# NOTE: If we parsed Element objects and looked at .sourceline,
|
||||||
|
# we'd be able to see the line numbers from the original document.
|
||||||
|
# But instead we build an XMLParser or HTMLParser object to serve
|
||||||
|
# as the target of parse messages, and those messages don't include
|
||||||
|
# line numbers.
|
||||||
|
# See: https://bugs.launchpad.net/lxml/+bug/1846906
|
||||||
|
|
||||||
|
def initialize_soup(self, soup):
|
||||||
|
"""Let the BeautifulSoup object know about the standard namespace
|
||||||
|
mapping.
|
||||||
|
|
||||||
|
:param soup: A `BeautifulSoup`.
|
||||||
|
"""
|
||||||
|
super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
|
||||||
|
self._register_namespaces(self.DEFAULT_NSMAPS)
|
||||||
|
|
||||||
|
def _register_namespaces(self, mapping):
|
||||||
|
"""Let the BeautifulSoup object know about namespaces encountered
|
||||||
|
while parsing the document.
|
||||||
|
|
||||||
|
This might be useful later on when creating CSS selectors.
|
||||||
|
|
||||||
|
This will track (almost) all namespaces, even ones that were
|
||||||
|
only in scope for part of the document. If two namespaces have
|
||||||
|
the same prefix, only the first one encountered will be
|
||||||
|
tracked. Un-prefixed namespaces are not tracked.
|
||||||
|
|
||||||
|
:param mapping: A dictionary mapping namespace prefixes to URIs.
|
||||||
|
"""
|
||||||
|
for key, value in list(mapping.items()):
|
||||||
|
# This is 'if key' and not 'if key is not None' because we
|
||||||
|
# don't track un-prefixed namespaces. Soupselect will
|
||||||
|
# treat an un-prefixed namespace as the default, which
|
||||||
|
# causes confusion in some cases.
|
||||||
|
if key and key not in self.soup._namespaces:
|
||||||
|
# Let the BeautifulSoup object know about a new namespace.
|
||||||
|
# If there are multiple namespaces defined with the same
|
||||||
|
# prefix, the first one in the document takes precedence.
|
||||||
|
self.soup._namespaces[key] = value
|
||||||
|
|
||||||
|
def default_parser(self, encoding):
|
||||||
|
"""Find the default parser for the given encoding.
|
||||||
|
|
||||||
|
:param encoding: A string.
|
||||||
|
:return: Either a parser object or a class, which
|
||||||
|
will be instantiated with default arguments.
|
||||||
|
"""
|
||||||
|
if self._default_parser is not None:
|
||||||
|
return self._default_parser
|
||||||
|
return etree.XMLParser(
|
||||||
|
target=self, strip_cdata=False, recover=True, encoding=encoding)
|
||||||
|
|
||||||
|
def parser_for(self, encoding):
|
||||||
|
"""Instantiate an appropriate parser for the given encoding.
|
||||||
|
|
||||||
|
:param encoding: A string.
|
||||||
|
:return: A parser object such as an `etree.XMLParser`.
|
||||||
|
"""
|
||||||
|
# Use the default parser.
|
||||||
|
parser = self.default_parser(encoding)
|
||||||
|
|
||||||
|
if isinstance(parser, Callable):
|
||||||
|
# Instantiate the parser with default arguments
|
||||||
|
parser = parser(
|
||||||
|
target=self, strip_cdata=False, recover=True, encoding=encoding
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
def __init__(self, parser=None, empty_element_tags=None, **kwargs):
|
||||||
|
# TODO: Issue a warning if parser is present but not a
|
||||||
|
# callable, since that means there's no way to create new
|
||||||
|
# parsers for different encodings.
|
||||||
|
self._default_parser = parser
|
||||||
|
if empty_element_tags is not None:
|
||||||
|
self.empty_element_tags = set(empty_element_tags)
|
||||||
|
self.soup = None
|
||||||
|
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
|
||||||
|
self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
|
||||||
|
super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
def _getNsTag(self, tag):
|
||||||
|
# Split the namespace URL out of a fully-qualified lxml tag
|
||||||
|
# name. Copied from lxml's src/lxml/sax.py.
|
||||||
|
if tag[0] == '{':
|
||||||
|
return tuple(tag[1:].split('}', 1))
|
||||||
|
else:
|
||||||
|
return (None, tag)
|
||||||
|
|
||||||
|
def prepare_markup(self, markup, user_specified_encoding=None,
|
||||||
|
exclude_encodings=None,
|
||||||
|
document_declared_encoding=None):
|
||||||
|
"""Run any preliminary steps necessary to make incoming markup
|
||||||
|
acceptable to the parser.
|
||||||
|
|
||||||
|
lxml really wants to get a bytestring and convert it to
|
||||||
|
Unicode itself. So instead of using UnicodeDammit to convert
|
||||||
|
the bytestring to Unicode using different encodings, this
|
||||||
|
implementation uses EncodingDetector to iterate over the
|
||||||
|
encodings, and tell lxml to try to parse the document as each
|
||||||
|
one in turn.
|
||||||
|
|
||||||
|
:param markup: Some markup -- hopefully a bytestring.
|
||||||
|
:param user_specified_encoding: The user asked to try this encoding.
|
||||||
|
:param document_declared_encoding: The markup itself claims to be
|
||||||
|
in this encoding.
|
||||||
|
:param exclude_encodings: The user asked _not_ to try any of
|
||||||
|
these encodings.
|
||||||
|
|
||||||
|
:yield: A series of 4-tuples:
|
||||||
|
(markup, encoding, declared encoding,
|
||||||
|
has undergone character replacement)
|
||||||
|
|
||||||
|
Each 4-tuple represents a strategy for converting the
|
||||||
|
document to Unicode and parsing it. Each strategy will be tried
|
||||||
|
in turn.
|
||||||
|
"""
|
||||||
|
is_html = not self.is_xml
|
||||||
|
if is_html:
|
||||||
|
self.processing_instruction_class = ProcessingInstruction
|
||||||
|
# We're in HTML mode, so if we're given XML, that's worth
|
||||||
|
# noting.
|
||||||
|
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
|
||||||
|
else:
|
||||||
|
self.processing_instruction_class = XMLProcessingInstruction
|
||||||
|
|
||||||
|
if isinstance(markup, str):
|
||||||
|
# We were given Unicode. Maybe lxml can parse Unicode on
|
||||||
|
# this system?
|
||||||
|
|
||||||
|
# TODO: This is a workaround for
|
||||||
|
# https://bugs.launchpad.net/lxml/+bug/1948551.
|
||||||
|
# We can remove it once the upstream issue is fixed.
|
||||||
|
if len(markup) > 0 and markup[0] == u'\N{BYTE ORDER MARK}':
|
||||||
|
markup = markup[1:]
|
||||||
|
yield markup, None, document_declared_encoding, False
|
||||||
|
|
||||||
|
if isinstance(markup, str):
|
||||||
|
# No, apparently not. Convert the Unicode to UTF-8 and
|
||||||
|
# tell lxml to parse it as UTF-8.
|
||||||
|
yield (markup.encode("utf8"), "utf8",
|
||||||
|
document_declared_encoding, False)
|
||||||
|
|
||||||
|
# This was provided by the end-user; treat it as a known
|
||||||
|
# definite encoding per the algorithm laid out in the HTML5
|
||||||
|
# spec. (See the EncodingDetector class for details.)
|
||||||
|
known_definite_encodings = [user_specified_encoding]
|
||||||
|
|
||||||
|
# This was found in the document; treat it as a slightly lower-priority
|
||||||
|
# user encoding.
|
||||||
|
user_encodings = [document_declared_encoding]
|
||||||
|
detector = EncodingDetector(
|
||||||
|
markup, known_definite_encodings=known_definite_encodings,
|
||||||
|
user_encodings=user_encodings, is_html=is_html,
|
||||||
|
exclude_encodings=exclude_encodings
|
||||||
|
)
|
||||||
|
for encoding in detector.encodings:
|
||||||
|
yield (detector.markup, encoding, document_declared_encoding, False)
|
||||||
|
|
||||||
|
def feed(self, markup):
|
||||||
|
if isinstance(markup, bytes):
|
||||||
|
markup = BytesIO(markup)
|
||||||
|
elif isinstance(markup, str):
|
||||||
|
markup = StringIO(markup)
|
||||||
|
|
||||||
|
# Call feed() at least once, even if the markup is empty,
|
||||||
|
# or the parser won't be initialized.
|
||||||
|
data = markup.read(self.CHUNK_SIZE)
|
||||||
|
try:
|
||||||
|
self.parser = self.parser_for(self.soup.original_encoding)
|
||||||
|
self.parser.feed(data)
|
||||||
|
while len(data) != 0:
|
||||||
|
# Now call feed() on the rest of the data, chunk by chunk.
|
||||||
|
data = markup.read(self.CHUNK_SIZE)
|
||||||
|
if len(data) != 0:
|
||||||
|
self.parser.feed(data)
|
||||||
|
self.parser.close()
|
||||||
|
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
|
||||||
|
raise ParserRejectedMarkup(e)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
|
||||||
|
|
||||||
|
def start(self, name, attrs, nsmap={}):
|
||||||
|
# Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
|
||||||
|
attrs = dict(attrs)
|
||||||
|
nsprefix = None
|
||||||
|
# Invert each namespace map as it comes in.
|
||||||
|
if len(nsmap) == 0 and len(self.nsmaps) > 1:
|
||||||
|
# There are no new namespaces for this tag, but
|
||||||
|
# non-default namespaces are in play, so we need a
|
||||||
|
# separate tag stack to know when they end.
|
||||||
|
self.nsmaps.append(None)
|
||||||
|
elif len(nsmap) > 0:
|
||||||
|
# A new namespace mapping has come into play.
|
||||||
|
|
||||||
|
# First, Let the BeautifulSoup object know about it.
|
||||||
|
self._register_namespaces(nsmap)
|
||||||
|
|
||||||
|
# Then, add it to our running list of inverted namespace
|
||||||
|
# mappings.
|
||||||
|
self.nsmaps.append(_invert(nsmap))
|
||||||
|
|
||||||
|
# The currently active namespace prefixes have
|
||||||
|
# changed. Calculate the new mapping so it can be stored
|
||||||
|
# with all Tag objects created while these prefixes are in
|
||||||
|
# scope.
|
||||||
|
current_mapping = dict(self.active_namespace_prefixes[-1])
|
||||||
|
current_mapping.update(nsmap)
|
||||||
|
|
||||||
|
# We should not track un-prefixed namespaces as we can only hold one
|
||||||
|
# and it will be recognized as the default namespace by soupsieve,
|
||||||
|
# which may be confusing in some situations.
|
||||||
|
if '' in current_mapping:
|
||||||
|
del current_mapping['']
|
||||||
|
self.active_namespace_prefixes.append(current_mapping)
|
||||||
|
|
||||||
|
# Also treat the namespace mapping as a set of attributes on the
|
||||||
|
# tag, so we can recreate it later.
|
||||||
|
attrs = attrs.copy()
|
||||||
|
for prefix, namespace in list(nsmap.items()):
|
||||||
|
attribute = NamespacedAttribute(
|
||||||
|
"xmlns", prefix, "http://www.w3.org/2000/xmlns/")
|
||||||
|
attrs[attribute] = namespace
|
||||||
|
|
||||||
|
# Namespaces are in play. Find any attributes that came in
|
||||||
|
# from lxml with namespaces attached to their names, and
|
||||||
|
# turn then into NamespacedAttribute objects.
|
||||||
|
new_attrs = {}
|
||||||
|
for attr, value in list(attrs.items()):
|
||||||
|
namespace, attr = self._getNsTag(attr)
|
||||||
|
if namespace is None:
|
||||||
|
new_attrs[attr] = value
|
||||||
|
else:
|
||||||
|
nsprefix = self._prefix_for_namespace(namespace)
|
||||||
|
attr = NamespacedAttribute(nsprefix, attr, namespace)
|
||||||
|
new_attrs[attr] = value
|
||||||
|
attrs = new_attrs
|
||||||
|
|
||||||
|
namespace, name = self._getNsTag(name)
|
||||||
|
nsprefix = self._prefix_for_namespace(namespace)
|
||||||
|
self.soup.handle_starttag(
|
||||||
|
name, namespace, nsprefix, attrs,
|
||||||
|
namespaces=self.active_namespace_prefixes[-1]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _prefix_for_namespace(self, namespace):
|
||||||
|
"""Find the currently active prefix for the given namespace."""
|
||||||
|
if namespace is None:
|
||||||
|
return None
|
||||||
|
for inverted_nsmap in reversed(self.nsmaps):
|
||||||
|
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||||
|
return inverted_nsmap[namespace]
|
||||||
|
return None
|
||||||
|
|
||||||
|
def end(self, name):
|
||||||
|
self.soup.endData()
|
||||||
|
completed_tag = self.soup.tagStack[-1]
|
||||||
|
namespace, name = self._getNsTag(name)
|
||||||
|
nsprefix = None
|
||||||
|
if namespace is not None:
|
||||||
|
for inverted_nsmap in reversed(self.nsmaps):
|
||||||
|
if inverted_nsmap is not None and namespace in inverted_nsmap:
|
||||||
|
nsprefix = inverted_nsmap[namespace]
|
||||||
|
break
|
||||||
|
self.soup.handle_endtag(name, nsprefix)
|
||||||
|
if len(self.nsmaps) > 1:
|
||||||
|
# This tag, or one of its parents, introduced a namespace
|
||||||
|
# mapping, so pop it off the stack.
|
||||||
|
out_of_scope_nsmap = self.nsmaps.pop()
|
||||||
|
|
||||||
|
if out_of_scope_nsmap is not None:
|
||||||
|
# This tag introduced a namespace mapping which is no
|
||||||
|
# longer in scope. Recalculate the currently active
|
||||||
|
# namespace prefixes.
|
||||||
|
self.active_namespace_prefixes.pop()
|
||||||
|
|
||||||
|
def pi(self, target, data):
|
||||||
|
self.soup.endData()
|
||||||
|
data = target + ' ' + data
|
||||||
|
self.soup.handle_data(data)
|
||||||
|
self.soup.endData(self.processing_instruction_class)
|
||||||
|
|
||||||
|
def data(self, content):
|
||||||
|
self.soup.handle_data(content)
|
||||||
|
|
||||||
|
def doctype(self, name, pubid, system):
|
||||||
|
self.soup.endData()
|
||||||
|
doctype = Doctype.for_name_and_ids(name, pubid, system)
|
||||||
|
self.soup.object_was_parsed(doctype)
|
||||||
|
|
||||||
|
def comment(self, content):
|
||||||
|
"Handle comments as Comment objects."
|
||||||
|
self.soup.endData()
|
||||||
|
self.soup.handle_data(content)
|
||||||
|
self.soup.endData(Comment)
|
||||||
|
|
||||||
|
def test_fragment_to_document(self, fragment):
|
||||||
|
"""See `TreeBuilder`."""
|
||||||
|
return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
|
||||||
|
|
||||||
|
|
||||||
|
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
|
||||||
|
|
||||||
|
NAME = LXML
|
||||||
|
ALTERNATE_NAMES = ["lxml-html"]
|
||||||
|
|
||||||
|
features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
|
||||||
|
is_xml = False
|
||||||
|
processing_instruction_class = ProcessingInstruction
|
||||||
|
|
||||||
|
def default_parser(self, encoding):
|
||||||
|
return etree.HTMLParser
|
||||||
|
|
||||||
|
def feed(self, markup):
|
||||||
|
encoding = self.soup.original_encoding
|
||||||
|
try:
|
||||||
|
self.parser = self.parser_for(encoding)
|
||||||
|
self.parser.feed(markup)
|
||||||
|
self.parser.close()
|
||||||
|
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
|
||||||
|
raise ParserRejectedMarkup(e)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fragment_to_document(self, fragment):
|
||||||
|
"""See `TreeBuilder`."""
|
||||||
|
return '<html><body>%s</body></html>' % fragment
|
1095
acer-env/lib/python3.10/site-packages/bs4/dammit.py
Normal file
1095
acer-env/lib/python3.10/site-packages/bs4/dammit.py
Normal file
File diff suppressed because it is too large
Load Diff
248
acer-env/lib/python3.10/site-packages/bs4/diagnose.py
Normal file
248
acer-env/lib/python3.10/site-packages/bs4/diagnose.py
Normal file
@ -0,0 +1,248 @@
|
|||||||
|
"""Diagnostic functions, mainly for use when doing tech support."""
|
||||||
|
|
||||||
|
# Use of this source code is governed by the MIT license.
|
||||||
|
__license__ = "MIT"
|
||||||
|
|
||||||
|
import cProfile
|
||||||
|
from io import BytesIO
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import bs4
|
||||||
|
from bs4 import BeautifulSoup, __version__
|
||||||
|
from bs4.builder import builder_registry
|
||||||
|
|
||||||
|
import os
|
||||||
|
import pstats
|
||||||
|
import random
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
import sys
|
||||||
|
import cProfile
|
||||||
|
|
||||||
|
def diagnose(data):
|
||||||
|
"""Diagnostic suite for isolating common problems.
|
||||||
|
|
||||||
|
:param data: A string containing markup that needs to be explained.
|
||||||
|
:return: None; diagnostics are printed to standard output.
|
||||||
|
"""
|
||||||
|
print(("Diagnostic running on Beautiful Soup %s" % __version__))
|
||||||
|
print(("Python version %s" % sys.version))
|
||||||
|
|
||||||
|
basic_parsers = ["html.parser", "html5lib", "lxml"]
|
||||||
|
for name in basic_parsers:
|
||||||
|
for builder in builder_registry.builders:
|
||||||
|
if name in builder.features:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
basic_parsers.remove(name)
|
||||||
|
print((
|
||||||
|
"I noticed that %s is not installed. Installing it may help." %
|
||||||
|
name))
|
||||||
|
|
||||||
|
if 'lxml' in basic_parsers:
|
||||||
|
basic_parsers.append("lxml-xml")
|
||||||
|
try:
|
||||||
|
from lxml import etree
|
||||||
|
print(("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))))
|
||||||
|
except ImportError as e:
|
||||||
|
print(
|
||||||
|
"lxml is not installed or couldn't be imported.")
|
||||||
|
|
||||||
|
|
||||||
|
if 'html5lib' in basic_parsers:
|
||||||
|
try:
|
||||||
|
import html5lib
|
||||||
|
print(("Found html5lib version %s" % html5lib.__version__))
|
||||||
|
except ImportError as e:
|
||||||
|
print(
|
||||||
|
"html5lib is not installed or couldn't be imported.")
|
||||||
|
|
||||||
|
if hasattr(data, 'read'):
|
||||||
|
data = data.read()
|
||||||
|
elif data.startswith("http:") or data.startswith("https:"):
|
||||||
|
print(('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data))
|
||||||
|
print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
if os.path.exists(data):
|
||||||
|
print(('"%s" looks like a filename. Reading data from the file.' % data))
|
||||||
|
with open(data) as fp:
|
||||||
|
data = fp.read()
|
||||||
|
except ValueError:
|
||||||
|
# This can happen on some platforms when the 'filename' is
|
||||||
|
# too long. Assume it's data and not a filename.
|
||||||
|
pass
|
||||||
|
print("")
|
||||||
|
|
||||||
|
for parser in basic_parsers:
|
||||||
|
print(("Trying to parse your markup with %s" % parser))
|
||||||
|
success = False
|
||||||
|
try:
|
||||||
|
soup = BeautifulSoup(data, features=parser)
|
||||||
|
success = True
|
||||||
|
except Exception as e:
|
||||||
|
print(("%s could not parse the markup." % parser))
|
||||||
|
traceback.print_exc()
|
||||||
|
if success:
|
||||||
|
print(("Here's what %s did with the markup:" % parser))
|
||||||
|
print((soup.prettify()))
|
||||||
|
|
||||||
|
print(("-" * 80))
|
||||||
|
|
||||||
|
def lxml_trace(data, html=True, **kwargs):
|
||||||
|
"""Print out the lxml events that occur during parsing.
|
||||||
|
|
||||||
|
This lets you see how lxml parses a document when no Beautiful
|
||||||
|
Soup code is running. You can use this to determine whether
|
||||||
|
an lxml-specific problem is in Beautiful Soup's lxml tree builders
|
||||||
|
or in lxml itself.
|
||||||
|
|
||||||
|
:param data: Some markup.
|
||||||
|
:param html: If True, markup will be parsed with lxml's HTML parser.
|
||||||
|
if False, lxml's XML parser will be used.
|
||||||
|
"""
|
||||||
|
from lxml import etree
|
||||||
|
recover = kwargs.pop('recover', True)
|
||||||
|
if isinstance(data, str):
|
||||||
|
data = data.encode("utf8")
|
||||||
|
reader = BytesIO(data)
|
||||||
|
for event, element in etree.iterparse(
|
||||||
|
reader, html=html, recover=recover, **kwargs
|
||||||
|
):
|
||||||
|
print(("%s, %4s, %s" % (event, element.tag, element.text)))
|
||||||
|
|
||||||
|
class AnnouncingParser(HTMLParser):
|
||||||
|
"""Subclass of HTMLParser that announces parse events, without doing
|
||||||
|
anything else.
|
||||||
|
|
||||||
|
You can use this to get a picture of how html.parser sees a given
|
||||||
|
document. The easiest way to do this is to call `htmlparser_trace`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _p(self, s):
|
||||||
|
print(s)
|
||||||
|
|
||||||
|
def handle_starttag(self, name, attrs):
|
||||||
|
self._p("%s START" % name)
|
||||||
|
|
||||||
|
def handle_endtag(self, name):
|
||||||
|
self._p("%s END" % name)
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
self._p("%s DATA" % data)
|
||||||
|
|
||||||
|
def handle_charref(self, name):
|
||||||
|
self._p("%s CHARREF" % name)
|
||||||
|
|
||||||
|
def handle_entityref(self, name):
|
||||||
|
self._p("%s ENTITYREF" % name)
|
||||||
|
|
||||||
|
def handle_comment(self, data):
|
||||||
|
self._p("%s COMMENT" % data)
|
||||||
|
|
||||||
|
def handle_decl(self, data):
|
||||||
|
self._p("%s DECL" % data)
|
||||||
|
|
||||||
|
def unknown_decl(self, data):
|
||||||
|
self._p("%s UNKNOWN-DECL" % data)
|
||||||
|
|
||||||
|
def handle_pi(self, data):
|
||||||
|
self._p("%s PI" % data)
|
||||||
|
|
||||||
|
def htmlparser_trace(data):
|
||||||
|
"""Print out the HTMLParser events that occur during parsing.
|
||||||
|
|
||||||
|
This lets you see how HTMLParser parses a document when no
|
||||||
|
Beautiful Soup code is running.
|
||||||
|
|
||||||
|
:param data: Some markup.
|
||||||
|
"""
|
||||||
|
parser = AnnouncingParser()
|
||||||
|
parser.feed(data)
|
||||||
|
|
||||||
|
_vowels = "aeiou"
|
||||||
|
_consonants = "bcdfghjklmnpqrstvwxyz"
|
||||||
|
|
||||||
|
def rword(length=5):
|
||||||
|
"Generate a random word-like string."
|
||||||
|
s = ''
|
||||||
|
for i in range(length):
|
||||||
|
if i % 2 == 0:
|
||||||
|
t = _consonants
|
||||||
|
else:
|
||||||
|
t = _vowels
|
||||||
|
s += random.choice(t)
|
||||||
|
return s
|
||||||
|
|
||||||
|
def rsentence(length=4):
|
||||||
|
"Generate a random sentence-like string."
|
||||||
|
return " ".join(rword(random.randint(4,9)) for i in range(length))
|
||||||
|
|
||||||
|
def rdoc(num_elements=1000):
|
||||||
|
"""Randomly generate an invalid HTML document."""
|
||||||
|
tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
|
||||||
|
elements = []
|
||||||
|
for i in range(num_elements):
|
||||||
|
choice = random.randint(0,3)
|
||||||
|
if choice == 0:
|
||||||
|
# New tag.
|
||||||
|
tag_name = random.choice(tag_names)
|
||||||
|
elements.append("<%s>" % tag_name)
|
||||||
|
elif choice == 1:
|
||||||
|
elements.append(rsentence(random.randint(1,4)))
|
||||||
|
elif choice == 2:
|
||||||
|
# Close a tag.
|
||||||
|
tag_name = random.choice(tag_names)
|
||||||
|
elements.append("</%s>" % tag_name)
|
||||||
|
return "<html>" + "\n".join(elements) + "</html>"
|
||||||
|
|
||||||
|
def benchmark_parsers(num_elements=100000):
|
||||||
|
"""Very basic head-to-head performance benchmark."""
|
||||||
|
print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
|
||||||
|
data = rdoc(num_elements)
|
||||||
|
print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
|
||||||
|
|
||||||
|
for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
|
||||||
|
success = False
|
||||||
|
try:
|
||||||
|
a = time.time()
|
||||||
|
soup = BeautifulSoup(data, parser)
|
||||||
|
b = time.time()
|
||||||
|
success = True
|
||||||
|
except Exception as e:
|
||||||
|
print(("%s could not parse the markup." % parser))
|
||||||
|
traceback.print_exc()
|
||||||
|
if success:
|
||||||
|
print(("BS4+%s parsed the markup in %.2fs." % (parser, b-a)))
|
||||||
|
|
||||||
|
from lxml import etree
|
||||||
|
a = time.time()
|
||||||
|
etree.HTML(data)
|
||||||
|
b = time.time()
|
||||||
|
print(("Raw lxml parsed the markup in %.2fs." % (b-a)))
|
||||||
|
|
||||||
|
import html5lib
|
||||||
|
parser = html5lib.HTMLParser()
|
||||||
|
a = time.time()
|
||||||
|
parser.parse(data)
|
||||||
|
b = time.time()
|
||||||
|
print(("Raw html5lib parsed the markup in %.2fs." % (b-a)))
|
||||||
|
|
||||||
|
def profile(num_elements=100000, parser="lxml"):
|
||||||
|
"""Use Python's profiler on a randomly generated document."""
|
||||||
|
filehandle = tempfile.NamedTemporaryFile()
|
||||||
|
filename = filehandle.name
|
||||||
|
|
||||||
|
data = rdoc(num_elements)
|
||||||
|
vars = dict(bs4=bs4, data=data, parser=parser)
|
||||||
|
cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)
|
||||||
|
|
||||||
|
stats = pstats.Stats(filename)
|
||||||
|
# stats.strip_dirs()
|
||||||
|
stats.sort_stats("cumulative")
|
||||||
|
stats.print_stats('_html5lib|bs4', 50)
|
||||||
|
|
||||||
|
# If this file is run as a script, standard input is diagnosed.
|
||||||
|
if __name__ == '__main__':
|
||||||
|
diagnose(sys.stdin.read())
|
2291
acer-env/lib/python3.10/site-packages/bs4/element.py
Normal file
2291
acer-env/lib/python3.10/site-packages/bs4/element.py
Normal file
File diff suppressed because it is too large
Load Diff
185
acer-env/lib/python3.10/site-packages/bs4/formatter.py
Normal file
185
acer-env/lib/python3.10/site-packages/bs4/formatter.py
Normal file
@ -0,0 +1,185 @@
|
|||||||
|
from bs4.dammit import EntitySubstitution
|
||||||
|
|
||||||
|
class Formatter(EntitySubstitution):
|
||||||
|
"""Describes a strategy to use when outputting a parse tree to a string.
|
||||||
|
|
||||||
|
Some parts of this strategy come from the distinction between
|
||||||
|
HTML4, HTML5, and XML. Others are configurable by the user.
|
||||||
|
|
||||||
|
Formatters are passed in as the `formatter` argument to methods
|
||||||
|
like `PageElement.encode`. Most people won't need to think about
|
||||||
|
formatters, and most people who need to think about them can pass
|
||||||
|
in one of these predefined strings as `formatter` rather than
|
||||||
|
making a new Formatter object:
|
||||||
|
|
||||||
|
For HTML documents:
|
||||||
|
* 'html' - HTML entity substitution for generic HTML documents. (default)
|
||||||
|
* 'html5' - HTML entity substitution for HTML5 documents, as
|
||||||
|
well as some optimizations in the way tags are rendered.
|
||||||
|
* 'minimal' - Only make the substitutions necessary to guarantee
|
||||||
|
valid HTML.
|
||||||
|
* None - Do not perform any substitution. This will be faster
|
||||||
|
but may result in invalid markup.
|
||||||
|
|
||||||
|
For XML documents:
|
||||||
|
* 'html' - Entity substitution for XHTML documents.
|
||||||
|
* 'minimal' - Only make the substitutions necessary to guarantee
|
||||||
|
valid XML. (default)
|
||||||
|
* None - Do not perform any substitution. This will be faster
|
||||||
|
but may result in invalid markup.
|
||||||
|
"""
|
||||||
|
# Registries of XML and HTML formatters.
|
||||||
|
XML_FORMATTERS = {}
|
||||||
|
HTML_FORMATTERS = {}
|
||||||
|
|
||||||
|
HTML = 'html'
|
||||||
|
XML = 'xml'
|
||||||
|
|
||||||
|
HTML_DEFAULTS = dict(
|
||||||
|
cdata_containing_tags=set(["script", "style"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _default(self, language, value, kwarg):
|
||||||
|
if value is not None:
|
||||||
|
return value
|
||||||
|
if language == self.XML:
|
||||||
|
return set()
|
||||||
|
return self.HTML_DEFAULTS[kwarg]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, language=None, entity_substitution=None,
|
||||||
|
void_element_close_prefix='/', cdata_containing_tags=None,
|
||||||
|
empty_attributes_are_booleans=False, indent=1,
|
||||||
|
):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param language: This should be Formatter.XML if you are formatting
|
||||||
|
XML markup and Formatter.HTML if you are formatting HTML markup.
|
||||||
|
|
||||||
|
:param entity_substitution: A function to call to replace special
|
||||||
|
characters with XML/HTML entities. For examples, see
|
||||||
|
bs4.dammit.EntitySubstitution.substitute_html and substitute_xml.
|
||||||
|
:param void_element_close_prefix: By default, void elements
|
||||||
|
are represented as <tag/> (XML rules) rather than <tag>
|
||||||
|
(HTML rules). To get <tag>, pass in the empty string.
|
||||||
|
:param cdata_containing_tags: The list of tags that are defined
|
||||||
|
as containing CDATA in this dialect. For example, in HTML,
|
||||||
|
<script> and <style> tags are defined as containing CDATA,
|
||||||
|
and their contents should not be formatted.
|
||||||
|
:param blank_attributes_are_booleans: Render attributes whose value
|
||||||
|
is the empty string as HTML-style boolean attributes.
|
||||||
|
(Attributes whose value is None are always rendered this way.)
|
||||||
|
|
||||||
|
:param indent: If indent is a non-negative integer or string,
|
||||||
|
then the contents of elements will be indented
|
||||||
|
appropriately when pretty-printing. An indent level of 0,
|
||||||
|
negative, or "" will only insert newlines. Using a
|
||||||
|
positive integer indent indents that many spaces per
|
||||||
|
level. If indent is a string (such as "\t"), that string
|
||||||
|
is used to indent each level. The default behavior to
|
||||||
|
indent one space per level.
|
||||||
|
"""
|
||||||
|
self.language = language
|
||||||
|
self.entity_substitution = entity_substitution
|
||||||
|
self.void_element_close_prefix = void_element_close_prefix
|
||||||
|
self.cdata_containing_tags = self._default(
|
||||||
|
language, cdata_containing_tags, 'cdata_containing_tags'
|
||||||
|
)
|
||||||
|
self.empty_attributes_are_booleans=empty_attributes_are_booleans
|
||||||
|
if indent is None:
|
||||||
|
indent = 0
|
||||||
|
if isinstance(indent, int):
|
||||||
|
if indent < 0:
|
||||||
|
indent = 0
|
||||||
|
indent = ' ' * indent
|
||||||
|
elif isinstance(indent, str):
|
||||||
|
indent = indent
|
||||||
|
else:
|
||||||
|
indent = ' '
|
||||||
|
self.indent = indent
|
||||||
|
|
||||||
|
def substitute(self, ns):
|
||||||
|
"""Process a string that needs to undergo entity substitution.
|
||||||
|
This may be a string encountered in an attribute value or as
|
||||||
|
text.
|
||||||
|
|
||||||
|
:param ns: A string.
|
||||||
|
:return: A string with certain characters replaced by named
|
||||||
|
or numeric entities.
|
||||||
|
"""
|
||||||
|
if not self.entity_substitution:
|
||||||
|
return ns
|
||||||
|
from .element import NavigableString
|
||||||
|
if (isinstance(ns, NavigableString)
|
||||||
|
and ns.parent is not None
|
||||||
|
and ns.parent.name in self.cdata_containing_tags):
|
||||||
|
# Do nothing.
|
||||||
|
return ns
|
||||||
|
# Substitute.
|
||||||
|
return self.entity_substitution(ns)
|
||||||
|
|
||||||
|
def attribute_value(self, value):
|
||||||
|
"""Process the value of an attribute.
|
||||||
|
|
||||||
|
:param ns: A string.
|
||||||
|
:return: A string with certain characters replaced by named
|
||||||
|
or numeric entities.
|
||||||
|
"""
|
||||||
|
return self.substitute(value)
|
||||||
|
|
||||||
|
def attributes(self, tag):
|
||||||
|
"""Reorder a tag's attributes however you want.
|
||||||
|
|
||||||
|
By default, attributes are sorted alphabetically. This makes
|
||||||
|
behavior consistent between Python 2 and Python 3, and preserves
|
||||||
|
backwards compatibility with older versions of Beautiful Soup.
|
||||||
|
|
||||||
|
If `empty_boolean_attributes` is True, then attributes whose
|
||||||
|
values are set to the empty string will be treated as boolean
|
||||||
|
attributes.
|
||||||
|
"""
|
||||||
|
if tag.attrs is None:
|
||||||
|
return []
|
||||||
|
return sorted(
|
||||||
|
(k, (None if self.empty_attributes_are_booleans and v == '' else v))
|
||||||
|
for k, v in list(tag.attrs.items())
|
||||||
|
)
|
||||||
|
|
||||||
|
class HTMLFormatter(Formatter):
|
||||||
|
"""A generic Formatter for HTML."""
|
||||||
|
REGISTRY = {}
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
return super(HTMLFormatter, self).__init__(self.HTML, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class XMLFormatter(Formatter):
|
||||||
|
"""A generic Formatter for XML."""
|
||||||
|
REGISTRY = {}
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
return super(XMLFormatter, self).__init__(self.XML, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
# Set up aliases for the default formatters.
|
||||||
|
HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
|
||||||
|
entity_substitution=EntitySubstitution.substitute_html
|
||||||
|
)
|
||||||
|
HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
|
||||||
|
entity_substitution=EntitySubstitution.substitute_html,
|
||||||
|
void_element_close_prefix=None,
|
||||||
|
empty_attributes_are_booleans=True,
|
||||||
|
)
|
||||||
|
HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
|
||||||
|
entity_substitution=EntitySubstitution.substitute_xml
|
||||||
|
)
|
||||||
|
HTMLFormatter.REGISTRY[None] = HTMLFormatter(
|
||||||
|
entity_substitution=None
|
||||||
|
)
|
||||||
|
XMLFormatter.REGISTRY["html"] = XMLFormatter(
|
||||||
|
entity_substitution=EntitySubstitution.substitute_html
|
||||||
|
)
|
||||||
|
XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
|
||||||
|
entity_substitution=EntitySubstitution.substitute_xml
|
||||||
|
)
|
||||||
|
XMLFormatter.REGISTRY[None] = Formatter(
|
||||||
|
Formatter(Formatter.XML, entity_substitution=None)
|
||||||
|
)
|
1191
acer-env/lib/python3.10/site-packages/bs4/tests/__init__.py
Normal file
1191
acer-env/lib/python3.10/site-packages/bs4/tests/__init__.py
Normal file
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,29 @@
|
|||||||
|
import pytest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from bs4.builder import DetectsXMLParsedAsHTML
|
||||||
|
|
||||||
|
class TestDetectsXMLParsedAsHTML(object):
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"markup,looks_like_xml",
|
||||||
|
[("No xml declaration", False),
|
||||||
|
("<html>obviously HTML</html", False),
|
||||||
|
("<?xml ><html>Actually XHTML</html>", False),
|
||||||
|
("<?xml> < html>Tricky XHTML</html>", False),
|
||||||
|
("<?xml ><no-html-tag>", True),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
def test_warn_if_markup_looks_like_xml(self, markup, looks_like_xml):
|
||||||
|
# Test of our ability to guess at whether markup looks XML-ish
|
||||||
|
# _and_ not HTML-ish.
|
||||||
|
with patch('bs4.builder.DetectsXMLParsedAsHTML._warn') as mock:
|
||||||
|
for data in markup, markup.encode('utf8'):
|
||||||
|
result = DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(
|
||||||
|
data
|
||||||
|
)
|
||||||
|
assert result == looks_like_xml
|
||||||
|
if looks_like_xml:
|
||||||
|
assert mock.called
|
||||||
|
else:
|
||||||
|
assert not mock.called
|
||||||
|
mock.reset_mock()
|
@ -0,0 +1,136 @@
|
|||||||
|
"""Tests of the builder registry."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from bs4.builder import (
|
||||||
|
builder_registry as registry,
|
||||||
|
HTMLParserTreeBuilder,
|
||||||
|
TreeBuilderRegistry,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from bs4.builder import HTML5TreeBuilder
|
||||||
|
HTML5LIB_PRESENT = True
|
||||||
|
except ImportError:
|
||||||
|
HTML5LIB_PRESENT = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
from bs4.builder import (
|
||||||
|
LXMLTreeBuilderForXML,
|
||||||
|
LXMLTreeBuilder,
|
||||||
|
)
|
||||||
|
LXML_PRESENT = True
|
||||||
|
except ImportError:
|
||||||
|
LXML_PRESENT = False
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuiltInRegistry(object):
|
||||||
|
"""Test the built-in registry with the default builders registered."""
|
||||||
|
|
||||||
|
def test_combination(self):
|
||||||
|
assert registry.lookup('strict', 'html') == HTMLParserTreeBuilder
|
||||||
|
if LXML_PRESENT:
|
||||||
|
assert registry.lookup('fast', 'html') == LXMLTreeBuilder
|
||||||
|
assert registry.lookup('permissive', 'xml') == LXMLTreeBuilderForXML
|
||||||
|
if HTML5LIB_PRESENT:
|
||||||
|
assert registry.lookup('html5lib', 'html') == HTML5TreeBuilder
|
||||||
|
|
||||||
|
def test_lookup_by_markup_type(self):
|
||||||
|
if LXML_PRESENT:
|
||||||
|
assert registry.lookup('html') == LXMLTreeBuilder
|
||||||
|
assert registry.lookup('xml') == LXMLTreeBuilderForXML
|
||||||
|
else:
|
||||||
|
assert registry.lookup('xml') == None
|
||||||
|
if HTML5LIB_PRESENT:
|
||||||
|
assert registry.lookup('html') == HTML5TreeBuilder
|
||||||
|
else:
|
||||||
|
assert registry.lookup('html') == HTMLParserTreeBuilder
|
||||||
|
|
||||||
|
def test_named_library(self):
|
||||||
|
if LXML_PRESENT:
|
||||||
|
assert registry.lookup('lxml', 'xml') == LXMLTreeBuilderForXML
|
||||||
|
assert registry.lookup('lxml', 'html') == LXMLTreeBuilder
|
||||||
|
if HTML5LIB_PRESENT:
|
||||||
|
assert registry.lookup('html5lib') == HTML5TreeBuilder
|
||||||
|
|
||||||
|
assert registry.lookup('html.parser') == HTMLParserTreeBuilder
|
||||||
|
|
||||||
|
def test_beautifulsoup_constructor_does_lookup(self):
|
||||||
|
|
||||||
|
with warnings.catch_warnings(record=True) as w:
|
||||||
|
# This will create a warning about not explicitly
|
||||||
|
# specifying a parser, but we'll ignore it.
|
||||||
|
|
||||||
|
# You can pass in a string.
|
||||||
|
BeautifulSoup("", features="html")
|
||||||
|
# Or a list of strings.
|
||||||
|
BeautifulSoup("", features=["html", "fast"])
|
||||||
|
pass
|
||||||
|
|
||||||
|
# You'll get an exception if BS can't find an appropriate
|
||||||
|
# builder.
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
BeautifulSoup("", features="no-such-feature")
|
||||||
|
|
||||||
|
class TestRegistry(object):
|
||||||
|
"""Test the TreeBuilderRegistry class in general."""
|
||||||
|
|
||||||
|
def setup_method(self):
|
||||||
|
self.registry = TreeBuilderRegistry()
|
||||||
|
|
||||||
|
def builder_for_features(self, *feature_list):
|
||||||
|
cls = type('Builder_' + '_'.join(feature_list),
|
||||||
|
(object,), {'features' : feature_list})
|
||||||
|
|
||||||
|
self.registry.register(cls)
|
||||||
|
return cls
|
||||||
|
|
||||||
|
def test_register_with_no_features(self):
|
||||||
|
builder = self.builder_for_features()
|
||||||
|
|
||||||
|
# Since the builder advertises no features, you can't find it
|
||||||
|
# by looking up features.
|
||||||
|
assert self.registry.lookup('foo') is None
|
||||||
|
|
||||||
|
# But you can find it by doing a lookup with no features, if
|
||||||
|
# this happens to be the only registered builder.
|
||||||
|
assert self.registry.lookup() == builder
|
||||||
|
|
||||||
|
def test_register_with_features_makes_lookup_succeed(self):
|
||||||
|
builder = self.builder_for_features('foo', 'bar')
|
||||||
|
assert self.registry.lookup('foo') is builder
|
||||||
|
assert self.registry.lookup('bar') is builder
|
||||||
|
|
||||||
|
def test_lookup_fails_when_no_builder_implements_feature(self):
|
||||||
|
builder = self.builder_for_features('foo', 'bar')
|
||||||
|
assert self.registry.lookup('baz') is None
|
||||||
|
|
||||||
|
def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
|
||||||
|
builder1 = self.builder_for_features('foo')
|
||||||
|
builder2 = self.builder_for_features('bar')
|
||||||
|
assert self.registry.lookup() == builder2
|
||||||
|
|
||||||
|
def test_lookup_fails_when_no_tree_builders_registered(self):
|
||||||
|
assert self.registry.lookup() is None
|
||||||
|
|
||||||
|
def test_lookup_gets_most_recent_builder_supporting_all_features(self):
|
||||||
|
has_one = self.builder_for_features('foo')
|
||||||
|
has_the_other = self.builder_for_features('bar')
|
||||||
|
has_both_early = self.builder_for_features('foo', 'bar', 'baz')
|
||||||
|
has_both_late = self.builder_for_features('foo', 'bar', 'quux')
|
||||||
|
lacks_one = self.builder_for_features('bar')
|
||||||
|
has_the_other = self.builder_for_features('foo')
|
||||||
|
|
||||||
|
# There are two builders featuring 'foo' and 'bar', but
|
||||||
|
# the one that also features 'quux' was registered later.
|
||||||
|
assert self.registry.lookup('foo', 'bar') == has_both_late
|
||||||
|
|
||||||
|
# There is only one builder featuring 'foo', 'bar', and 'baz'.
|
||||||
|
assert self.registry.lookup('foo', 'bar', 'baz') == has_both_early
|
||||||
|
|
||||||
|
def test_lookup_fails_when_cannot_reconcile_requested_features(self):
|
||||||
|
builder1 = self.builder_for_features('foo', 'bar')
|
||||||
|
builder2 = self.builder_for_features('foo', 'baz')
|
||||||
|
assert self.registry.lookup('bar', 'baz') is None
|
371
acer-env/lib/python3.10/site-packages/bs4/tests/test_dammit.py
Normal file
371
acer-env/lib/python3.10/site-packages/bs4/tests/test_dammit.py
Normal file
@ -0,0 +1,371 @@
|
|||||||
|
# encoding: utf-8
|
||||||
|
import pytest
|
||||||
|
import logging
|
||||||
|
import bs4
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from bs4.dammit import (
|
||||||
|
EntitySubstitution,
|
||||||
|
EncodingDetector,
|
||||||
|
UnicodeDammit,
|
||||||
|
)
|
||||||
|
|
||||||
|
class TestUnicodeDammit(object):
|
||||||
|
"""Standalone tests of UnicodeDammit."""
|
||||||
|
|
||||||
|
def test_unicode_input(self):
|
||||||
|
markup = "I'm already Unicode! \N{SNOWMAN}"
|
||||||
|
dammit = UnicodeDammit(markup)
|
||||||
|
assert dammit.unicode_markup == markup
|
||||||
|
|
||||||
|
def test_smart_quotes_to_unicode(self):
|
||||||
|
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||||
|
dammit = UnicodeDammit(markup)
|
||||||
|
assert dammit.unicode_markup == "<foo>\u2018\u2019\u201c\u201d</foo>"
|
||||||
|
|
||||||
|
def test_smart_quotes_to_xml_entities(self):
|
||||||
|
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||||
|
dammit = UnicodeDammit(markup, smart_quotes_to="xml")
|
||||||
|
assert dammit.unicode_markup == "<foo>‘’“”</foo>"
|
||||||
|
|
||||||
|
def test_smart_quotes_to_html_entities(self):
|
||||||
|
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||||
|
dammit = UnicodeDammit(markup, smart_quotes_to="html")
|
||||||
|
assert dammit.unicode_markup == "<foo>‘’“”</foo>"
|
||||||
|
|
||||||
|
def test_smart_quotes_to_ascii(self):
|
||||||
|
markup = b"<foo>\x91\x92\x93\x94</foo>"
|
||||||
|
dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
|
||||||
|
assert dammit.unicode_markup == """<foo>''""</foo>"""
|
||||||
|
|
||||||
|
def test_detect_utf8(self):
|
||||||
|
utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
|
||||||
|
dammit = UnicodeDammit(utf8)
|
||||||
|
assert dammit.original_encoding.lower() == 'utf-8'
|
||||||
|
assert dammit.unicode_markup == 'Sacr\xe9 bleu! \N{SNOWMAN}'
|
||||||
|
|
||||||
|
def test_convert_hebrew(self):
|
||||||
|
hebrew = b"\xed\xe5\xec\xf9"
|
||||||
|
dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
|
||||||
|
assert dammit.original_encoding.lower() == 'iso-8859-8'
|
||||||
|
assert dammit.unicode_markup == '\u05dd\u05d5\u05dc\u05e9'
|
||||||
|
|
||||||
|
def test_dont_see_smart_quotes_where_there_are_none(self):
|
||||||
|
utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
|
||||||
|
dammit = UnicodeDammit(utf_8)
|
||||||
|
assert dammit.original_encoding.lower() == 'utf-8'
|
||||||
|
assert dammit.unicode_markup.encode("utf-8") == utf_8
|
||||||
|
|
||||||
|
def test_ignore_inappropriate_codecs(self):
|
||||||
|
utf8_data = "Räksmörgås".encode("utf-8")
|
||||||
|
dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
|
||||||
|
assert dammit.original_encoding.lower() == 'utf-8'
|
||||||
|
|
||||||
|
def test_ignore_invalid_codecs(self):
|
||||||
|
utf8_data = "Räksmörgås".encode("utf-8")
|
||||||
|
for bad_encoding in ['.utf8', '...', 'utF---16.!']:
|
||||||
|
dammit = UnicodeDammit(utf8_data, [bad_encoding])
|
||||||
|
assert dammit.original_encoding.lower() == 'utf-8'
|
||||||
|
|
||||||
|
def test_exclude_encodings(self):
|
||||||
|
# This is UTF-8.
|
||||||
|
utf8_data = "Räksmörgås".encode("utf-8")
|
||||||
|
|
||||||
|
# But if we exclude UTF-8 from consideration, the guess is
|
||||||
|
# Windows-1252.
|
||||||
|
dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
|
||||||
|
assert dammit.original_encoding.lower() == 'windows-1252'
|
||||||
|
|
||||||
|
# And if we exclude that, there is no valid guess at all.
|
||||||
|
dammit = UnicodeDammit(
|
||||||
|
utf8_data, exclude_encodings=["utf-8", "windows-1252"])
|
||||||
|
assert dammit.original_encoding == None
|
||||||
|
|
||||||
|
class TestEncodingDetector(object):
|
||||||
|
|
||||||
|
def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
|
||||||
|
detected = EncodingDetector(
|
||||||
|
b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
|
||||||
|
encodings = list(detected.encodings)
|
||||||
|
assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings
|
||||||
|
|
||||||
|
def test_detect_html5_style_meta_tag(self):
|
||||||
|
|
||||||
|
for data in (
|
||||||
|
b'<html><meta charset="euc-jp" /></html>',
|
||||||
|
b"<html><meta charset='euc-jp' /></html>",
|
||||||
|
b"<html><meta charset=euc-jp /></html>",
|
||||||
|
b"<html><meta charset=euc-jp/></html>"):
|
||||||
|
dammit = UnicodeDammit(data, is_html=True)
|
||||||
|
assert "euc-jp" == dammit.original_encoding
|
||||||
|
|
||||||
|
def test_last_ditch_entity_replacement(self):
|
||||||
|
# This is a UTF-8 document that contains bytestrings
|
||||||
|
# completely incompatible with UTF-8 (ie. encoded with some other
|
||||||
|
# encoding).
|
||||||
|
#
|
||||||
|
# Since there is no consistent encoding for the document,
|
||||||
|
# Unicode, Dammit will eventually encode the document as UTF-8
|
||||||
|
# and encode the incompatible characters as REPLACEMENT
|
||||||
|
# CHARACTER.
|
||||||
|
#
|
||||||
|
# If chardet is installed, it will detect that the document
|
||||||
|
# can be converted into ISO-8859-1 without errors. This happens
|
||||||
|
# to be the wrong encoding, but it is a consistent encoding, so the
|
||||||
|
# code we're testing here won't run.
|
||||||
|
#
|
||||||
|
# So we temporarily disable chardet if it's present.
|
||||||
|
doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<html><b>\330\250\330\252\330\261</b>
|
||||||
|
<i>\310\322\321\220\312\321\355\344</i></html>"""
|
||||||
|
chardet = bs4.dammit.chardet_dammit
|
||||||
|
logging.disable(logging.WARNING)
|
||||||
|
try:
|
||||||
|
def noop(str):
|
||||||
|
return None
|
||||||
|
bs4.dammit.chardet_dammit = noop
|
||||||
|
dammit = UnicodeDammit(doc)
|
||||||
|
assert True == dammit.contains_replacement_characters
|
||||||
|
assert "\ufffd" in dammit.unicode_markup
|
||||||
|
|
||||||
|
soup = BeautifulSoup(doc, "html.parser")
|
||||||
|
assert soup.contains_replacement_characters
|
||||||
|
finally:
|
||||||
|
logging.disable(logging.NOTSET)
|
||||||
|
bs4.dammit.chardet_dammit = chardet
|
||||||
|
|
||||||
|
def test_byte_order_mark_removed(self):
|
||||||
|
# A document written in UTF-16LE will have its byte order marker stripped.
|
||||||
|
data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
|
||||||
|
dammit = UnicodeDammit(data)
|
||||||
|
assert "<a>áé</a>" == dammit.unicode_markup
|
||||||
|
assert "utf-16le" == dammit.original_encoding
|
||||||
|
|
||||||
|
def test_known_definite_versus_user_encodings(self):
|
||||||
|
# The known_definite_encodings are used before sniffing the
|
||||||
|
# byte-order mark; the user_encodings are used afterwards.
|
||||||
|
|
||||||
|
# Here's a document in UTF-16LE.
|
||||||
|
data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
|
||||||
|
dammit = UnicodeDammit(data)
|
||||||
|
|
||||||
|
# We can process it as UTF-16 by passing it in as a known
|
||||||
|
# definite encoding.
|
||||||
|
before = UnicodeDammit(data, known_definite_encodings=["utf-16"])
|
||||||
|
assert "utf-16" == before.original_encoding
|
||||||
|
|
||||||
|
# If we pass UTF-18 as a user encoding, it's not even
|
||||||
|
# tried--the encoding sniffed from the byte-order mark takes
|
||||||
|
# precedence.
|
||||||
|
after = UnicodeDammit(data, user_encodings=["utf-8"])
|
||||||
|
assert "utf-16le" == after.original_encoding
|
||||||
|
assert ["utf-16le"] == [x[0] for x in dammit.tried_encodings]
|
||||||
|
|
||||||
|
# Here's a document in ISO-8859-8.
|
||||||
|
hebrew = b"\xed\xe5\xec\xf9"
|
||||||
|
dammit = UnicodeDammit(hebrew, known_definite_encodings=["utf-8"],
|
||||||
|
user_encodings=["iso-8859-8"])
|
||||||
|
|
||||||
|
# The known_definite_encodings don't work, BOM sniffing does
|
||||||
|
# nothing (it only works for a few UTF encodings), but one of
|
||||||
|
# the user_encodings does work.
|
||||||
|
assert "iso-8859-8" == dammit.original_encoding
|
||||||
|
assert ["utf-8", "iso-8859-8"] == [x[0] for x in dammit.tried_encodings]
|
||||||
|
|
||||||
|
def test_deprecated_override_encodings(self):
|
||||||
|
# override_encodings is a deprecated alias for
|
||||||
|
# known_definite_encodings.
|
||||||
|
hebrew = b"\xed\xe5\xec\xf9"
|
||||||
|
dammit = UnicodeDammit(
|
||||||
|
hebrew,
|
||||||
|
known_definite_encodings=["shift-jis"],
|
||||||
|
override_encodings=["utf-8"],
|
||||||
|
user_encodings=["iso-8859-8"],
|
||||||
|
)
|
||||||
|
assert "iso-8859-8" == dammit.original_encoding
|
||||||
|
|
||||||
|
# known_definite_encodings and override_encodings were tried
|
||||||
|
# before user_encodings.
|
||||||
|
assert ["shift-jis", "utf-8", "iso-8859-8"] == (
|
||||||
|
[x[0] for x in dammit.tried_encodings]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_detwingle(self):
|
||||||
|
# Here's a UTF8 document.
|
||||||
|
utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")
|
||||||
|
|
||||||
|
# Here's a Windows-1252 document.
|
||||||
|
windows_1252 = (
|
||||||
|
"\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
|
||||||
|
"\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
|
||||||
|
|
||||||
|
# Through some unholy alchemy, they've been stuck together.
|
||||||
|
doc = utf8 + windows_1252 + utf8
|
||||||
|
|
||||||
|
# The document can't be turned into UTF-8:
|
||||||
|
with pytest.raises(UnicodeDecodeError):
|
||||||
|
doc.decode("utf8")
|
||||||
|
|
||||||
|
# Unicode, Dammit thinks the whole document is Windows-1252,
|
||||||
|
# and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃"
|
||||||
|
|
||||||
|
# But if we run it through fix_embedded_windows_1252, it's fixed:
|
||||||
|
fixed = UnicodeDammit.detwingle(doc)
|
||||||
|
assert "☃☃☃“Hi, I like Windows!”☃☃☃" == fixed.decode("utf8")
|
||||||
|
|
||||||
|
def test_detwingle_ignores_multibyte_characters(self):
|
||||||
|
# Each of these characters has a UTF-8 representation ending
|
||||||
|
# in \x93. \x93 is a smart quote if interpreted as
|
||||||
|
# Windows-1252. But our code knows to skip over multibyte
|
||||||
|
# UTF-8 characters, so they'll survive the process unscathed.
|
||||||
|
for tricky_unicode_char in (
|
||||||
|
"\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
|
||||||
|
"\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
|
||||||
|
"\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
|
||||||
|
):
|
||||||
|
input = tricky_unicode_char.encode("utf8")
|
||||||
|
assert input.endswith(b'\x93')
|
||||||
|
output = UnicodeDammit.detwingle(input)
|
||||||
|
assert output == input
|
||||||
|
|
||||||
|
def test_find_declared_encoding(self):
|
||||||
|
# Test our ability to find a declared encoding inside an
|
||||||
|
# XML or HTML document.
|
||||||
|
#
|
||||||
|
# Even if the document comes in as Unicode, it may be
|
||||||
|
# interesting to know what encoding was claimed
|
||||||
|
# originally.
|
||||||
|
|
||||||
|
html_unicode = '<html><head><meta charset="utf-8"></head></html>'
|
||||||
|
html_bytes = html_unicode.encode("ascii")
|
||||||
|
|
||||||
|
xml_unicode= '<?xml version="1.0" encoding="ISO-8859-1" ?>'
|
||||||
|
xml_bytes = xml_unicode.encode("ascii")
|
||||||
|
|
||||||
|
m = EncodingDetector.find_declared_encoding
|
||||||
|
assert m(html_unicode, is_html=False) is None
|
||||||
|
assert "utf-8" == m(html_unicode, is_html=True)
|
||||||
|
assert "utf-8" == m(html_bytes, is_html=True)
|
||||||
|
|
||||||
|
assert "iso-8859-1" == m(xml_unicode)
|
||||||
|
assert "iso-8859-1" == m(xml_bytes)
|
||||||
|
|
||||||
|
# Normally, only the first few kilobytes of a document are checked for
|
||||||
|
# an encoding.
|
||||||
|
spacer = b' ' * 5000
|
||||||
|
assert m(spacer + html_bytes) is None
|
||||||
|
assert m(spacer + xml_bytes) is None
|
||||||
|
|
||||||
|
# But you can tell find_declared_encoding to search an entire
|
||||||
|
# HTML document.
|
||||||
|
assert (
|
||||||
|
m(spacer + html_bytes, is_html=True, search_entire_document=True)
|
||||||
|
== "utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
# The XML encoding declaration has to be the very first thing
|
||||||
|
# in the document. We'll allow whitespace before the document
|
||||||
|
# starts, but nothing else.
|
||||||
|
assert m(xml_bytes, search_entire_document=True) == "iso-8859-1"
|
||||||
|
assert m(b' ' + xml_bytes, search_entire_document=True) == "iso-8859-1"
|
||||||
|
assert m(b'a' + xml_bytes, search_entire_document=True) is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestEntitySubstitution(object):
|
||||||
|
"""Standalone tests of the EntitySubstitution class."""
|
||||||
|
def setup_method(self):
|
||||||
|
self.sub = EntitySubstitution
|
||||||
|
|
||||||
|
def test_simple_html_substitution(self):
|
||||||
|
# Unicode characters corresponding to named HTML entites
|
||||||
|
# are substituted, and no others.
|
||||||
|
s = "foo\u2200\N{SNOWMAN}\u00f5bar"
|
||||||
|
assert self.sub.substitute_html(s) == "foo∀\N{SNOWMAN}õbar"
|
||||||
|
|
||||||
|
def test_smart_quote_substitution(self):
|
||||||
|
# MS smart quotes are a common source of frustration, so we
|
||||||
|
# give them a special test.
|
||||||
|
quotes = b"\x91\x92foo\x93\x94"
|
||||||
|
dammit = UnicodeDammit(quotes)
|
||||||
|
assert self.sub.substitute_html(dammit.markup) == "‘’foo“”"
|
||||||
|
|
||||||
|
def test_html5_entity(self):
|
||||||
|
# Some HTML5 entities correspond to single- or multi-character
|
||||||
|
# Unicode sequences.
|
||||||
|
|
||||||
|
for entity, u in (
|
||||||
|
# A few spot checks of our ability to recognize
|
||||||
|
# special character sequences and convert them
|
||||||
|
# to named entities.
|
||||||
|
('⊧', '\u22a7'),
|
||||||
|
('𝔑', '\U0001d511'),
|
||||||
|
('≧̸', '\u2267\u0338'),
|
||||||
|
('¬', '\xac'),
|
||||||
|
('⫬', '\u2aec'),
|
||||||
|
|
||||||
|
# We _could_ convert | to &verbarr;, but we don't, because
|
||||||
|
# | is an ASCII character.
|
||||||
|
('|' '|'),
|
||||||
|
|
||||||
|
# Similarly for the fj ligature, which we could convert to
|
||||||
|
# fj, but we don't.
|
||||||
|
("fj", "fj"),
|
||||||
|
|
||||||
|
# We do convert _these_ ASCII characters to HTML entities,
|
||||||
|
# because that's required to generate valid HTML.
|
||||||
|
('>', '>'),
|
||||||
|
('<', '<'),
|
||||||
|
('&', '&'),
|
||||||
|
):
|
||||||
|
template = '3 %s 4'
|
||||||
|
raw = template % u
|
||||||
|
with_entities = template % entity
|
||||||
|
assert self.sub.substitute_html(raw) == with_entities
|
||||||
|
|
||||||
|
def test_html5_entity_with_variation_selector(self):
|
||||||
|
# Some HTML5 entities correspond either to a single-character
|
||||||
|
# Unicode sequence _or_ to the same character plus U+FE00,
|
||||||
|
# VARIATION SELECTOR 1. We can handle this.
|
||||||
|
data = "fjords \u2294 penguins"
|
||||||
|
markup = "fjords ⊔ penguins"
|
||||||
|
assert self.sub.substitute_html(data) == markup
|
||||||
|
|
||||||
|
data = "fjords \u2294\ufe00 penguins"
|
||||||
|
markup = "fjords ⊔︀ penguins"
|
||||||
|
assert self.sub.substitute_html(data) == markup
|
||||||
|
|
||||||
|
def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
|
||||||
|
s = 'Welcome to "my bar"'
|
||||||
|
assert self.sub.substitute_xml(s, False) == s
|
||||||
|
|
||||||
|
def test_xml_attribute_quoting_normally_uses_double_quotes(self):
|
||||||
|
assert self.sub.substitute_xml("Welcome", True) == '"Welcome"'
|
||||||
|
assert self.sub.substitute_xml("Bob's Bar", True) == '"Bob\'s Bar"'
|
||||||
|
|
||||||
|
def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
|
||||||
|
s = 'Welcome to "my bar"'
|
||||||
|
assert self.sub.substitute_xml(s, True) == "'Welcome to \"my bar\"'"
|
||||||
|
|
||||||
|
def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
|
||||||
|
s = 'Welcome to "Bob\'s Bar"'
|
||||||
|
assert self.sub.substitute_xml(s, True) == '"Welcome to "Bob\'s Bar""'
|
||||||
|
|
||||||
|
def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
|
||||||
|
quoted = 'Welcome to "Bob\'s Bar"'
|
||||||
|
assert self.sub.substitute_xml(quoted) == quoted
|
||||||
|
|
||||||
|
def test_xml_quoting_handles_angle_brackets(self):
|
||||||
|
assert self.sub.substitute_xml("foo<bar>") == "foo<bar>"
|
||||||
|
|
||||||
|
def test_xml_quoting_handles_ampersands(self):
|
||||||
|
assert self.sub.substitute_xml("AT&T") == "AT&T"
|
||||||
|
|
||||||
|
def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
|
||||||
|
assert self.sub.substitute_xml("ÁT&T") == "&Aacute;T&T"
|
||||||
|
|
||||||
|
def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
|
||||||
|
assert self.sub.substitute_xml_containing_entities("ÁT&T") == "ÁT&T"
|
||||||
|
|
||||||
|
def test_quotes_not_html_substituted(self):
|
||||||
|
"""There's no need to do this except inside attribute values."""
|
||||||
|
text = 'Bob\'s "bar"'
|
||||||
|
assert self.sub.substitute_html(text) == text
|
38
acer-env/lib/python3.10/site-packages/bs4/tests/test_docs.py
Normal file
38
acer-env/lib/python3.10/site-packages/bs4/tests/test_docs.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
"Test harness for doctests."
|
||||||
|
|
||||||
|
# TODO: Pretty sure this isn't used and should be deleted.
|
||||||
|
|
||||||
|
# pylint: disable-msg=E0611,W0142
|
||||||
|
|
||||||
|
__metaclass__ = type
|
||||||
|
__all__ = [
|
||||||
|
'additional_tests',
|
||||||
|
]
|
||||||
|
|
||||||
|
import atexit
|
||||||
|
import doctest
|
||||||
|
import os
|
||||||
|
#from pkg_resources import (
|
||||||
|
# resource_filename, resource_exists, resource_listdir, cleanup_resources)
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
DOCTEST_FLAGS = (
|
||||||
|
doctest.ELLIPSIS |
|
||||||
|
doctest.NORMALIZE_WHITESPACE |
|
||||||
|
doctest.REPORT_NDIFF)
|
||||||
|
|
||||||
|
|
||||||
|
# def additional_tests():
|
||||||
|
# "Run the doc tests (README.txt and docs/*, if any exist)"
|
||||||
|
# doctest_files = [
|
||||||
|
# os.path.abspath(resource_filename('bs4', 'README.txt'))]
|
||||||
|
# if resource_exists('bs4', 'docs'):
|
||||||
|
# for name in resource_listdir('bs4', 'docs'):
|
||||||
|
# if name.endswith('.txt'):
|
||||||
|
# doctest_files.append(
|
||||||
|
# os.path.abspath(
|
||||||
|
# resource_filename('bs4', 'docs/%s' % name)))
|
||||||
|
# kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
|
||||||
|
# atexit.register(cleanup_resources)
|
||||||
|
# return unittest.TestSuite((
|
||||||
|
# doctest.DocFileSuite(*doctest_files, **kwargs)))
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user