From adad0a67beffb829fa89b5810c800596c0fa8c9f Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Apr 2022 11:42:42 +0545 Subject: [PATCH 01/45] lambda functions updated with sentry --- lambda_fns/entry_predict/app.py | 7 +++++++ lambda_fns/entry_predict/requirements.txt | 3 ++- lambda_fns/entry_predict_output_request/app.py | 7 +++++++ lambda_fns/entry_predict_output_request/requirements.txt | 3 ++- lambda_fns/extract_docs/app.py | 7 +++++++ lambda_fns/extract_docs/requirements.txt | 3 ++- lambda_fns/extract_docs_output_request/app.py | 7 +++++++ lambda_fns/extract_docs_output_request/requirements.txt | 3 ++- 8 files changed, 36 insertions(+), 4 deletions(-) diff --git a/lambda_fns/entry_predict/app.py b/lambda_fns/entry_predict/app.py index e50fdded..319c7c10 100644 --- a/lambda_fns/entry_predict/app.py +++ b/lambda_fns/entry_predict/app.py @@ -6,6 +6,8 @@ import logging from concurrent.futures import ThreadPoolExecutor, as_completed +import sentry_sdk + from postprocess_raw_preds import get_predictions_all, get_clean_thresholds, get_clean_ratios logging.getLogger().setLevel(logging.INFO) @@ -19,6 +21,9 @@ RELIABILITY_FN_NAME = os.environ.get("RELIABILITY_FN_NAME") MODEL_INFO_FN_NAME = os.environ.get("MODEL_INFO_FN_NAME") +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + sqs_client = boto3.client('sqs', region_name=AWS_REGION) sagemaker_rt = boto3.client("runtime.sagemaker", region_name="us-east-1") # todo: update the region later. @@ -26,6 +31,8 @@ reliability_client = boto3.client("lambda", region_name="us-east-1") model_info_client = boto3.client("lambda", region_name=AWS_REGION) +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) + class PredictionStatus(Enum): FAILED = 0 diff --git a/lambda_fns/entry_predict/requirements.txt b/lambda_fns/entry_predict/requirements.txt index 4f5b8993..a8661b56 100644 --- a/lambda_fns/entry_predict/requirements.txt +++ b/lambda_fns/entry_predict/requirements.txt @@ -1 +1,2 @@ -requests==2.26.0 \ No newline at end of file +requests==2.26.0 +sentry-sdk==1.5.8 \ No newline at end of file diff --git a/lambda_fns/entry_predict_output_request/app.py b/lambda_fns/entry_predict_output_request/app.py index 721446c5..ffb3d2fd 100644 --- a/lambda_fns/entry_predict_output_request/app.py +++ b/lambda_fns/entry_predict_output_request/app.py @@ -1,6 +1,8 @@ +import os import requests import json import logging +import sentry_sdk from mappings.tags_mapping import get_all_mappings, get_categories try: from lambda_fns.model_info.app import lambda_handler @@ -8,6 +10,11 @@ except ImportError: pass +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) + logging.getLogger().setLevel(logging.INFO) mappings = get_all_mappings() diff --git a/lambda_fns/entry_predict_output_request/requirements.txt b/lambda_fns/entry_predict_output_request/requirements.txt index 4f5b8993..a8661b56 100644 --- a/lambda_fns/entry_predict_output_request/requirements.txt +++ b/lambda_fns/entry_predict_output_request/requirements.txt @@ -1 +1,2 @@ -requests==2.26.0 \ No newline at end of file +requests==2.26.0 +sentry-sdk==1.5.8 \ No newline at end of file diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 337c258b..41df9fe0 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -12,6 +12,8 @@ import tempfile import signal +import sentry_sdk + from deep_parser import TextFromFile from deep_parser import TextFromWeb @@ -32,12 +34,17 @@ domain_name = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + s3_client = boto3.client('s3', region_name=aws_region) sqs_client = boto3.client('sqs', region_name=aws_region) lambda_client = boto3.client('lambda', region_name="us-east-1") extract_content_type = ExtractContentType() +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) + class ExtractionStatus(Enum): FAILED = 0 diff --git a/lambda_fns/extract_docs/requirements.txt b/lambda_fns/extract_docs/requirements.txt index b7d263c1..a016f5d9 100644 --- a/lambda_fns/extract_docs/requirements.txt +++ b/lambda_fns/extract_docs/requirements.txt @@ -10,4 +10,5 @@ six==1.16.0 urllib3==1.26.6 Pillow==8.3.2 lxml==4.7.1 -deep-parser @ git+https://github.com/the-deep/deepExt \ No newline at end of file +deep-parser @ git+https://github.com/the-deep/deepExt +sentry-sdk==1.5.8 \ No newline at end of file diff --git a/lambda_fns/extract_docs_output_request/app.py b/lambda_fns/extract_docs_output_request/app.py index d4166b55..7123f848 100644 --- a/lambda_fns/extract_docs_output_request/app.py +++ b/lambda_fns/extract_docs_output_request/app.py @@ -6,6 +6,8 @@ from botocore.exceptions import ClientError from botocore.client import Config +import sentry_sdk + logging.getLogger().setLevel(logging.INFO) REQUEST_TIMEOUT = 60 @@ -14,6 +16,11 @@ aws_region = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) signed_url_expiry_secs = os.environ.get("SIGNED_URL_EXPIRY_SECS") +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) + s3_client = boto3.client( 's3', region_name=aws_region, diff --git a/lambda_fns/extract_docs_output_request/requirements.txt b/lambda_fns/extract_docs_output_request/requirements.txt index 4f5b8993..a8661b56 100644 --- a/lambda_fns/extract_docs_output_request/requirements.txt +++ b/lambda_fns/extract_docs_output_request/requirements.txt @@ -1 +1,2 @@ -requests==2.26.0 \ No newline at end of file +requests==2.26.0 +sentry-sdk==1.5.8 \ No newline at end of file From 423a13a220dd381cfdf6dd77f7d5e3071f32cf96 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Apr 2022 11:52:39 +0545 Subject: [PATCH 02/45] terraform configs updated with sentry_url passed as env variable; updated the configs in gateway for prod, staging envs; --- main.tf | 9 +++++++ modules/api_gateway/main.tf | 7 +++--- modules/api_gateway/variables.tf | 1 + modules/reserved_lambda_entry_predict/main.tf | 25 ++++++++++++------- .../variables.tf | 4 ++- .../reserved_sqs_lambda_extract_docs/main.tf | 24 ++++++++++-------- .../variables.tf | 4 ++- modules/sqs_lambda_entry_predict/main.tf | 8 ++++++ modules/sqs_lambda_entry_predict/variables.tf | 4 ++- modules/sqs_lambda_extract_docs/main.tf | 6 ++++- modules/sqs_lambda_extract_docs/variables.tf | 4 ++- variables.tf | 6 ++++- 12 files changed, 74 insertions(+), 28 deletions(-) diff --git a/main.tf b/main.tf index 38f4ed20..8f124655 100644 --- a/main.tf +++ b/main.tf @@ -44,6 +44,8 @@ module "sqs_lambda_module" { reserved_input_queue_arn = "${module.reserved_sqs_lambda_module.reserved_input_queue_arn}" environment = var.environment + + sentry_url = var.sentry_url } module "reserved_sqs_lambda_module" { @@ -57,6 +59,8 @@ module "reserved_sqs_lambda_module" { docs_convert_lambda_fn_name = var.docs_convert_lambda_fn_name environment = var.environment + + sentry_url = var.sentry_url } module "sqs_lambda_predict_module" { @@ -73,6 +77,8 @@ module "sqs_lambda_predict_module" { aws_region = var.aws_region environment = var.environment + + sentry_url = var.sentry_url } module "reserved_sqs_lambda_predict_module" { @@ -86,6 +92,8 @@ module "reserved_sqs_lambda_predict_module" { aws_region = var.aws_region environment = var.environment + + sentry_url = var.sentry_url } module "apigateway_module" { @@ -94,6 +102,7 @@ module "apigateway_module" { api_gateway_name = var.api_gateway_name vpce_id = var.vpce_id + vpc_id = var.vpc_id predict_entry_invoke_arn = "${module.sqs_lambda_predict_module.entry_input_pred_request_predict_invoke_arn}" process_doc_invoke_arn = "${module.sqs_lambda_module.extract_doc_invoke_arn}" diff --git a/modules/api_gateway/main.tf b/modules/api_gateway/main.tf index 3b4f06da..2e60f135 100644 --- a/modules/api_gateway/main.tf +++ b/modules/api_gateway/main.tf @@ -137,10 +137,11 @@ resource "aws_api_gateway_rest_api_policy" "api_policy" { Effect = "Deny", Principal = "*", Action = "execute-api:Invoke", - Resource = "execute-api:/*/*/*", + Resource = "execute-api:/${var.environment}/*/*", Condition = { StringNotEquals = { - "aws:sourceVpce": var.vpce_id + "aws:sourceVpce": var.vpce_id, + "aws:sourceVpc": var.vpc_id } } }, @@ -148,7 +149,7 @@ resource "aws_api_gateway_rest_api_policy" "api_policy" { Effect = "Allow", Principal = "*", Action = "execute-api:Invoke", - Resource = "execute-api:/*/*/*" + Resource = "execute-api:/${var.environment}/*/*" } ] }) diff --git a/modules/api_gateway/variables.tf b/modules/api_gateway/variables.tf index 0f631683..f7bde9d9 100644 --- a/modules/api_gateway/variables.tf +++ b/modules/api_gateway/variables.tf @@ -23,6 +23,7 @@ variable "input_te_lambda_fn_alias_arn" {} variable "input_te_lambda_fn_alias_name" {} variable "vpce_id" {} +variable "vpc_id" {} variable "predict_entry_invoke_arn" {} diff --git a/modules/reserved_lambda_entry_predict/main.tf b/modules/reserved_lambda_entry_predict/main.tf index 9e782061..ebfc6924 100644 --- a/modules/reserved_lambda_entry_predict/main.tf +++ b/modules/reserved_lambda_entry_predict/main.tf @@ -131,7 +131,7 @@ module "reserved_predict_entry_fn" { ] }) - provisioned_concurrent_executions = 10 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 10 reserved_concurrent_executions = 30 environment_variables = { @@ -140,12 +140,14 @@ module "reserved_predict_entry_fn" { GEOLOCATION_FN_NAME = var.geolocation_fn_name RELIABILITY_FN_NAME = var.reliability_fn_name MODEL_INFO_FN_NAME = "${var.model_info_fn_name}-${var.environment}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } resource "aws_appautoscaling_target" "reserved_predict_entry_fn_autoscale" { - max_capacity = 10 - min_capacity = 5 + max_capacity = var.environment == "dev" ? 1 : 10 + min_capacity = var.environment == "dev" ? 1 : 5 resource_id = "function:${module.reserved_predict_entry_fn.lambda_function_name}:${module.reserved_predict_entry_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -200,13 +202,18 @@ module "reserved_entry_predict_output_fn" { ] }) - provisioned_concurrent_executions = 5 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 layers = ["${aws_lambda_layer_version.reserved_lambda_layer_mappings.arn}"] build_in_docker = true #store_on_s3 = true #s3_bucket = "${var.processed_docs_bucket}" + + environment_variables = { + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" + } } resource "aws_lambda_layer_version" "reserved_lambda_layer_mappings" { @@ -218,8 +225,8 @@ resource "aws_lambda_layer_version" "reserved_lambda_layer_mappings" { } resource "aws_appautoscaling_target" "reserved_entry_predict_output_fn_autoscale" { - max_capacity = 5 - min_capacity = 2 + max_capacity = var.environment == "dev" ? 1 : 5 + min_capacity = var.environment == "dev" ? 1 : 2 resource_id = "function:${module.reserved_entry_predict_output_fn.lambda_function_name}:${module.reserved_entry_predict_output_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -266,7 +273,7 @@ module "reserved_entry_predict_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = 5 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 environment_variables = { PREDICTION_QUEUE = aws_sqs_queue.reserved_entry_input_processed_queue_predict.id @@ -274,8 +281,8 @@ module "reserved_entry_predict_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_entry_predict_transfer_dlq_msg_autoscale" { - max_capacity = 5 - min_capacity = 2 + max_capacity = var.environment == "dev" ? 1 : 5 + min_capacity = var.environment == "dev" ? 1 : 2 resource_id = "function:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_name}:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_lambda_entry_predict/variables.tf b/modules/reserved_lambda_entry_predict/variables.tf index caa3db1e..d02dcd0b 100644 --- a/modules/reserved_lambda_entry_predict/variables.tf +++ b/modules/reserved_lambda_entry_predict/variables.tf @@ -10,4 +10,6 @@ variable reliability_fn_name {} variable model_info_fn_name {} -variable processed_docs_bucket {} \ No newline at end of file +variable processed_docs_bucket {} + +variable sentry_url {} \ No newline at end of file diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index 19fa3342..62fc6f7e 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -131,15 +131,17 @@ module "reserved_extract_docs_fn" { ] }) - provisioned_concurrent_executions = 10 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 10 reserved_concurrent_executions = 30 build_in_docker = true environment_variables = { INPUT_QUEUE = aws_sqs_queue.reserved_input_queue.id DEST_S3_BUCKET = "${var.processed_docs_bucket}" - PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id, + PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id DOCS_CONVERT_LAMBDA_FN_NAME = "${var.docs_convert_lambda_fn_name}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } @@ -156,8 +158,8 @@ module "reserved_extract_docs_fn" { # } resource "aws_appautoscaling_target" "reserved_extract_docs_autoscale" { - max_capacity = 10 - min_capacity = 5 + max_capacity = var.environment == "dev" ? 1 : 10 + min_capacity = var.environment == "dev" ? 1 : 5 resource_id = "function:${module.reserved_extract_docs_fn.lambda_function_name}:${module.reserved_extract_docs_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -204,7 +206,7 @@ module "reserved_output_request_fn" { ] }) - provisioned_concurrent_executions = 5 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 build_in_docker = true #store_on_s3 = true @@ -212,12 +214,14 @@ module "reserved_output_request_fn" { environment_variables = { SIGNED_URL_EXPIRY_SECS = "${var.signed_url_expiry_secs}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } resource "aws_appautoscaling_target" "reserved_output_fn_autoscale" { - max_capacity = 5 - min_capacity = 2 + max_capacity = var.environment == "dev" ? 1 : 5 + min_capacity = var.environment == "dev" ? 1 : 2 resource_id = "function:${module.reserved_output_request_fn.lambda_function_name}:${module.reserved_output_request_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -265,7 +269,7 @@ module "reserved_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = 5 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 environment_variables = { PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id @@ -273,8 +277,8 @@ module "reserved_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_transfer_dlq_msg_autoscale" { - max_capacity = 5 - min_capacity = 2 + max_capacity = var.environment == "dev" ? 1 : 5 + min_capacity = var.environment == "dev" ? 1 : 2 resource_id = "function:${module.reserved_transfer_dlq_msg.lambda_function_name}:${module.reserved_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_sqs_lambda_extract_docs/variables.tf b/modules/reserved_sqs_lambda_extract_docs/variables.tf index 7d28e964..2b183b0b 100644 --- a/modules/reserved_sqs_lambda_extract_docs/variables.tf +++ b/modules/reserved_sqs_lambda_extract_docs/variables.tf @@ -13,4 +13,6 @@ variable "processed_docs_bucket_arn" {} variable "docs_extract_fn_image_name" {} -variable "docs_convert_lambda_fn_name" {} \ No newline at end of file +variable "docs_convert_lambda_fn_name" {} + +variable sentry_url {} \ No newline at end of file diff --git a/modules/sqs_lambda_entry_predict/main.tf b/modules/sqs_lambda_entry_predict/main.tf index 6693d3ff..7c944d2d 100644 --- a/modules/sqs_lambda_entry_predict/main.tf +++ b/modules/sqs_lambda_entry_predict/main.tf @@ -106,6 +106,7 @@ module "predict_entry_fn" { source_path = [ { path = "${path.module}/../../lambda_fns/entry_predict" + pip_requirements = "${path.module}/../../lambda_fns/entry_predict/requirements.txt" } ] @@ -145,6 +146,8 @@ module "predict_entry_fn" { GEOLOCATION_FN_NAME = var.geolocation_fn_name RELIABILITY_FN_NAME = var.reliability_fn_name MODEL_INFO_FN_NAME = "${var.model_info_fn_name}-${var.environment}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } @@ -200,6 +203,11 @@ module "entry_predict_output_fn" { build_in_docker = true #store_on_s3 = true #s3_bucket = "${var.processed_docs_bucket}" + + environment_variables = { + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" + } } resource "aws_lambda_layer_version" "lambda_layer_mappings" { diff --git a/modules/sqs_lambda_entry_predict/variables.tf b/modules/sqs_lambda_entry_predict/variables.tf index 3ba01b47..2fd12100 100644 --- a/modules/sqs_lambda_entry_predict/variables.tf +++ b/modules/sqs_lambda_entry_predict/variables.tf @@ -14,4 +14,6 @@ variable processed_docs_bucket {} variable reserved_entry_input_queue_predict_id {} -variable reserved_entry_input_queue_predict_arn {} \ No newline at end of file +variable reserved_entry_input_queue_predict_arn {} + +variable sentry_url {} \ No newline at end of file diff --git a/modules/sqs_lambda_extract_docs/main.tf b/modules/sqs_lambda_extract_docs/main.tf index 98ead9ca..73c8b620 100644 --- a/modules/sqs_lambda_extract_docs/main.tf +++ b/modules/sqs_lambda_extract_docs/main.tf @@ -143,8 +143,10 @@ module "extract_docs_fn" { environment_variables = { INPUT_QUEUE = aws_sqs_queue.input_queue.id DEST_S3_BUCKET = "${var.processed_docs_bucket}" - PROCESSED_QUEUE = aws_sqs_queue.processed_queue.id, + PROCESSED_QUEUE = aws_sqs_queue.processed_queue.id DOCS_CONVERT_LAMBDA_FN_NAME = "${var.docs_convert_lambda_fn_name}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } @@ -194,6 +196,8 @@ module "output_request_fn" { environment_variables = { SIGNED_URL_EXPIRY_SECS = "${var.signed_url_expiry_secs}" + ENVIRONMENT = "${var.environment}" + SENTRY_URL = "${var.sentry_url}" } } diff --git a/modules/sqs_lambda_extract_docs/variables.tf b/modules/sqs_lambda_extract_docs/variables.tf index 421d90b8..7bf2f820 100644 --- a/modules/sqs_lambda_extract_docs/variables.tf +++ b/modules/sqs_lambda_extract_docs/variables.tf @@ -16,4 +16,6 @@ variable "docs_extract_fn_image_name" {} variable "docs_convert_lambda_fn_name" {} variable reserved_input_queue_id {} -variable reserved_input_queue_arn {} \ No newline at end of file +variable reserved_input_queue_arn {} + +variable sentry_url {} \ No newline at end of file diff --git a/variables.tf b/variables.tf index a061186b..fadc34df 100644 --- a/variables.tf +++ b/variables.tf @@ -8,6 +8,8 @@ variable api_gateway_name {} variable vpce_id {} +variable vpc_id {} + variable model_endpoint_name {} variable geolocation_fn_name {} @@ -18,4 +20,6 @@ variable model_info_fn_name {} variable docs_extract_fn_image_name {} -variable docs_convert_lambda_fn_name {} \ No newline at end of file +variable docs_convert_lambda_fn_name {} + +variable sentry_url {} \ No newline at end of file From 8235f168796f558c8677aa1caf166c609dddbf3a Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Apr 2022 11:54:08 +0545 Subject: [PATCH 03/45] values updated for dev, staging, prod environments in their variable files; --- dev.tfvars | 6 +++++- prod.tfvars | 10 +++++++--- staging.tfvars | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 staging.tfvars diff --git a/dev.tfvars b/dev.tfvars index 663b308e..60c204ed 100644 --- a/dev.tfvars +++ b/dev.tfvars @@ -5,6 +5,7 @@ environment = "dev" # api gateway api_gateway_name = "rapi" vpce_id = "vpce-02c7bb08b571074e1" +vpc_id = "vpc-0e65245d5e4c2deaf" # models model_endpoint_name = "test-all-models-rsh" @@ -16,4 +17,7 @@ model_info_fn_name = "model_info" docs_extract_fn_image_name = "extractor-tool" # docs convert lambda -docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" \ No newline at end of file +docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" + +# sentry url +sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file diff --git a/prod.tfvars b/prod.tfvars index f6a31e95..02be5089 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -1,10 +1,11 @@ -aws_region = "eu-west-3" +aws_region = "us-east-1" aws_profile = "default" environment = "prod" # api gateway api_gateway_name = "rapi" -vpce_id = "vpce-000796c803825026c" +vpce_id = "vpce-05d8c268ef4c0c443" +vpc_id = "vpc-0947f040a9d4692a7" # models model_endpoint_name = "test-all-models-rsh" @@ -16,4 +17,7 @@ model_info_fn_name = "model_info" docs_extract_fn_image_name = "extract-tool" # docs convert lambda -docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" \ No newline at end of file +docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" + +# sentry url +sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file diff --git a/staging.tfvars b/staging.tfvars new file mode 100644 index 00000000..9f7b840b --- /dev/null +++ b/staging.tfvars @@ -0,0 +1,23 @@ +aws_region = "us-east-1" +aws_profile = "default" +environment = "staging" + +# api gateway +api_gateway_name = "rapi" +vpce_id = "vpce-02c7bb08b571074e1" +vpc_id = "vpc-0e65245d5e4c2deaf" + +# models +model_endpoint_name = "test-all-models-rsh" +geolocation_fn_name = "geolocations" +reliability_fn_name = "reliability" +model_info_fn_name = "model_info" + +# ecr image name +docs_extract_fn_image_name = "extract-tool" + +# docs convert lambda +docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" + +# sentry url +sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file From 150791be840ab82428fa28efa50ba81782f7dbd2 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Apr 2022 16:29:47 +0545 Subject: [PATCH 04/45] added the missing package dependencies in terraform config; --- .github/workflows/terraform-plan.yml | 1 + modules/reserved_lambda_entry_predict/main.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index bd6e95ed..8facf22d 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -4,6 +4,7 @@ on: push: branches: - initial_setup + - feat/sentry-integration jobs: terraform: diff --git a/modules/reserved_lambda_entry_predict/main.tf b/modules/reserved_lambda_entry_predict/main.tf index ebfc6924..a8c96fc6 100644 --- a/modules/reserved_lambda_entry_predict/main.tf +++ b/modules/reserved_lambda_entry_predict/main.tf @@ -98,6 +98,7 @@ module "reserved_predict_entry_fn" { source_path = [ { path = "${path.module}/../../lambda_fns/entry_predict" + pip_requirements = "${path.module}/../../lambda_fns/entry_predict/requirements.txt" } ] From 1e6b69ea9ce6f2013df267cbad7a5c1a870326ca Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 10:19:32 +0545 Subject: [PATCH 05/45] updated the parser tool url; --- lambda_fns/extract_docs/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda_fns/extract_docs/requirements.txt b/lambda_fns/extract_docs/requirements.txt index a016f5d9..4eb5689c 100644 --- a/lambda_fns/extract_docs/requirements.txt +++ b/lambda_fns/extract_docs/requirements.txt @@ -10,5 +10,5 @@ six==1.16.0 urllib3==1.26.6 Pillow==8.3.2 lxml==4.7.1 -deep-parser @ git+https://github.com/the-deep/deepExt -sentry-sdk==1.5.8 \ No newline at end of file +sentry-sdk==1.5.8 +deep-parser @ git+https://github.com/the-deep/deepex From f16855605c9e2a8fd275eda9c1459341db7fe5ee Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 11:44:59 +0545 Subject: [PATCH 06/45] enabled stacktrace in sentry; handles image links(and discards them, no extraction happens); --- lambda_fns/entry_predict/app.py | 2 +- lambda_fns/entry_predict_output_request/app.py | 2 +- lambda_fns/extract_docs/app.py | 6 +++++- lambda_fns/extract_docs/content_types.py | 5 +++++ lambda_fns/extract_docs_output_request/app.py | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lambda_fns/entry_predict/app.py b/lambda_fns/entry_predict/app.py index 319c7c10..e4d9da91 100644 --- a/lambda_fns/entry_predict/app.py +++ b/lambda_fns/entry_predict/app.py @@ -31,7 +31,7 @@ reliability_client = boto3.client("lambda", region_name="us-east-1") model_info_client = boto3.client("lambda", region_name=AWS_REGION) -sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) class PredictionStatus(Enum): diff --git a/lambda_fns/entry_predict_output_request/app.py b/lambda_fns/entry_predict_output_request/app.py index ffb3d2fd..b2a5f90b 100644 --- a/lambda_fns/entry_predict_output_request/app.py +++ b/lambda_fns/entry_predict_output_request/app.py @@ -13,7 +13,7 @@ SENTRY_URL = os.environ.get("SENTRY_URL") ENVIRONMENT = os.environ.get("ENVIRONMENT") -sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) logging.getLogger().setLevel(logging.INFO) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 41df9fe0..7c351853 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -43,7 +43,7 @@ extract_content_type = ExtractContentType() -sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) class ExtractionStatus(Enum): @@ -346,6 +346,10 @@ def handle_urls(url, mock=False): logging.error(f"Exception occurred {e}") s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value + elif content_type == UrlTypes.IMG.value: + logging.warn("Text extraction from Images is not available.") + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value else: raise NotImplementedError diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index 533f818f..6f7f37d2 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -14,6 +14,7 @@ class UrlTypes(str, Enum): MSWORD = 'doc' XLSX = 'xlsx' XLS = 'xls' + IMG = 'img' class ExtractContentType: @@ -27,6 +28,7 @@ def __init__(self): self.content_types_ppt = ('application/vnd.ms-powerpoint') self.content_types_xlsx = ('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') self.content_types_xls = ('application/vnd.ms-excel') + self.content_types_img = ('image/jpeg', 'image/gif', 'image/png', 'image/svg+xml', 'image/webp', 'image/bmp', 'image/tiff') def get_content_type(self, url): try: @@ -53,6 +55,9 @@ def get_content_type(self, url): return UrlTypes.PPTX.value elif url.endswith(".ppt") or content_type in self.content_types_ppt: return UrlTypes.PPT.value + elif url.endswith(".jpg") or url.endswith(".jpeg") or url.endswith(".png") or \ + url.endswith(".gif") or url.endswith(".bmp") or content_type in self.content_types_img: + return UrlTypes.IMG.value else: logging.warn(f'Could not determine the content-type of the {url}') return None diff --git a/lambda_fns/extract_docs_output_request/app.py b/lambda_fns/extract_docs_output_request/app.py index 7123f848..8ba82606 100644 --- a/lambda_fns/extract_docs_output_request/app.py +++ b/lambda_fns/extract_docs_output_request/app.py @@ -19,7 +19,7 @@ SENTRY_URL = os.environ.get("SENTRY_URL") ENVIRONMENT = os.environ.get("ENVIRONMENT") -sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, traces_sample_rate=1.0) +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) s3_client = boto3.client( 's3', From 74e353ad55b6030604553a795a000c4440aa1603 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 15:10:53 +0545 Subject: [PATCH 07/45] enabled err stack; --- lambda_fns/extract_docs/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 7c351853..0a750919 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -399,7 +399,7 @@ def process_docs(event, context): } send_message2sqs(**sqs_message) except Exception as e: - logging.error(f"Exception is {e}") + logging.error(e, exc_info=True) signal.alarm(0) From 9e09b2a22e438c230224bda4a2f9e182ad9b514a Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 16:41:14 +0545 Subject: [PATCH 08/45] updated the hashicorp aws version; --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 8f124655..00b5d9f7 100644 --- a/main.tf +++ b/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "4.8.0" + version = "4.9.0" } } required_version = "1.1.2" From 5eb386212c9043be6803abeff9629c6011b0dc67 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Apr 2022 11:52:39 +0545 Subject: [PATCH 09/45] terraform configs updated with sentry_url passed as env variable; updated the configs in gateway for prod, staging envs; From 4f98fa5d87d44b847e6d3e379bfe488ed768397c Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 15:10:53 +0545 Subject: [PATCH 10/45] enabled err stack; From 54e86e73c927d7b1ad416ec9c03f13374bf6f202 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 28 Apr 2022 16:41:14 +0545 Subject: [PATCH 11/45] updated the hashicorp aws version; From 4274ac80858390f9411f04b0d93da8d61dee1fe5 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 5 May 2022 11:35:57 +0545 Subject: [PATCH 12/45] if content type is not determined with head requests, downloads the file to find the extension of that file; --- lambda_fns/extract_docs/content_types.py | 30 +- lambda_fns/extract_docs/wget.py | 404 +++++++++++++++++++++++ 2 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 lambda_fns/extract_docs/wget.py diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index 6f7f37d2..7144baaa 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -1,6 +1,11 @@ +import os import requests from enum import Enum import logging +try: + from wget import download +except ImportError: + from .wget import download logging.getLogger().setLevel(logging.INFO) @@ -59,8 +64,29 @@ def get_content_type(self, url): url.endswith(".gif") or url.endswith(".bmp") or content_type in self.content_types_img: return UrlTypes.IMG.value else: - logging.warn(f'Could not determine the content-type of the {url}') - return None + temp_filepath = download(url, out="/tmp/") + if os.path.exists(temp_filepath): + os.remove(temp_filepath) + if temp_filepath.endswith(".pdf"): + return UrlTypes.PDF.value + elif temp_filepath.endswith(".docx"): + return UrlTypes.DOCX.value + elif temp_filepath.endswith(".doc"): + return UrlTypes.MSWORD.value + elif temp_filepath.endswith(".xlsx"): + return UrlTypes.XLSX.value + elif temp_filepath.endswith(".xls"): + return UrlTypes.XLS.value + elif temp_filepath.endswith(".pptx"): + return UrlTypes.PPTX.value + elif temp_filepath.endswith(".ppt"): + return UrlTypes.PPT.value + elif temp_filepath.endswith(".jpg") or temp_filepath.endswith(".jpeg") or temp_filepath.endswith(".png") or \ + temp_filepath.endswith(".gif") or temp_filepath.endswith(".bmp"): + return UrlTypes.IMG.value + else: + logging.warn(f'Could not determine the content-type of the {url}') + return None except requests.exceptions.RequestException: logging.error(f'Exception occurred. Could not determine the content-type of the {url}') return None diff --git a/lambda_fns/extract_docs/wget.py b/lambda_fns/extract_docs/wget.py new file mode 100644 index 00000000..04b31916 --- /dev/null +++ b/lambda_fns/extract_docs/wget.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +""" +Download utility as an easy way to get file from the net + + python -m wget + python wget.py + +Downloads: http://pypi.python.org/pypi/wget/ +Development: http://bitbucket.org/techtonik/python-wget/ + +wget.py is not option compatible with Unix wget utility, +to make command line interface intuitive for new people. + +Public domain by anatoly techtonik +Also available under the terms of MIT license +Copyright (c) 2010-2014 anatoly techtonik + +** Modified by Ranjan Shrestha Line number 302 to make it work on aws lambda function ** +""" + + +import sys, shutil, os +import tempfile +import math + +PY3K = sys.version_info >= (3, 0) +if PY3K: + import urllib.request as urllib + import urllib.parse as urlparse +else: + import urllib + import urlparse + + +__version__ = "2.3-beta1" + + +def filename_from_url(url): + """:return: detected filename or None""" + fname = os.path.basename(urlparse.urlparse(url).path) + if len(fname.strip(" \n\t.")) == 0: + return None + return fname + +def filename_from_headers(headers): + """Detect filename from Content-Disposition headers if present. + http://greenbytes.de/tech/tc2231/ + + :param: headers as dict, list or string + :return: filename from content-disposition header or None + """ + if type(headers) == str: + headers = headers.splitlines() + if type(headers) == list: + headers = dict([x.split(':', 1) for x in headers]) + cdisp = headers.get("Content-Disposition") + if not cdisp: + return None + cdtype = cdisp.split(';') + if len(cdtype) == 1: + return None + if cdtype[0].strip().lower() not in ('inline', 'attachment'): + return None + # several filename params is illegal, but just in case + fnames = [x for x in cdtype[1:] if x.strip().startswith('filename=')] + if len(fnames) > 1: + return None + name = fnames[0].split('=')[1].strip(' \t"') + name = os.path.basename(name) + if not name: + return None + return name + +def filename_fix_existing(filename): + """Expands name portion of filename with numeric ' (x)' suffix to + return filename that doesn't exist already. + """ + dirname = '.' + name, ext = filename.rsplit('.', 1) + names = [x for x in os.listdir(dirname) if x.startswith(name)] + names = [x.rsplit('.', 1)[0] for x in names] + suffixes = [x.replace(name, '') for x in names] + # filter suffixes that match ' (x)' pattern + suffixes = [x[2:-1] for x in suffixes + if x.startswith(' (') and x.endswith(')')] + indexes = [int(x) for x in suffixes + if set(x) <= set('0123456789')] + idx = 1 + if indexes: + idx += sorted(indexes)[-1] + return '%s (%d).%s' % (name, idx, ext) + + +# --- terminal/console output helpers --- + +def get_console_width(): + """Return width of available window area. Autodetection works for + Windows and POSIX platforms. Returns 80 for others + + Code from http://bitbucket.org/techtonik/python-pager + """ + + if os.name == 'nt': + STD_INPUT_HANDLE = -10 + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + + # get console handle + from ctypes import windll, Structure, byref + try: + from ctypes.wintypes import SHORT, WORD, DWORD + except ImportError: + # workaround for missing types in Python 2.5 + from ctypes import ( + c_short as SHORT, c_ushort as WORD, c_ulong as DWORD) + console_handle = windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) + + # CONSOLE_SCREEN_BUFFER_INFO Structure + class COORD(Structure): + _fields_ = [("X", SHORT), ("Y", SHORT)] + + class SMALL_RECT(Structure): + _fields_ = [("Left", SHORT), ("Top", SHORT), + ("Right", SHORT), ("Bottom", SHORT)] + + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [("dwSize", COORD), + ("dwCursorPosition", COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", DWORD)] + + sbi = CONSOLE_SCREEN_BUFFER_INFO() + ret = windll.kernel32.GetConsoleScreenBufferInfo(console_handle, byref(sbi)) + if ret == 0: + return 0 + return sbi.srWindow.Right+1 + + elif os.name == 'posix': + from fcntl import ioctl + from termios import TIOCGWINSZ + from array import array + + winsize = array("H", [0] * 4) + try: + ioctl(sys.stdout.fileno(), TIOCGWINSZ, winsize) + except IOError: + pass + return (winsize[1], winsize[0])[0] + + return 80 + + +def bar_thermometer(current, total, width=80): + """Return thermometer style progress bar string. `total` argument + can not be zero. The minimum size of bar returned is 3. Example: + + [.......... ] + + Control and trailing symbols (\r and spaces) are not included. + See `bar_adaptive` for more information. + """ + # number of dots on thermometer scale + avail_dots = width-2 + shaded_dots = int(math.floor(float(current) / total * avail_dots)) + return '[' + '.'*shaded_dots + ' '*(avail_dots-shaded_dots) + ']' + +def bar_adaptive(current, total, width=80): + """Return progress bar string for given values in one of three + styles depending on available width: + + [.. ] downloaded / total + downloaded / total + [.. ] + + if total value is unknown or <= 0, show bytes counter using two + adaptive styles: + + %s / unknown + %s + + if there is not enough space on the screen, do not display anything + + returned string doesn't include control characters like \r used to + place cursor at the beginning of the line to erase previous content. + + this function leaves one free character at the end of string to + avoid automatic linefeed on Windows. + """ + + # process special case when total size is unknown and return immediately + if not total or total < 0: + msg = "%s / unknown" % current + if len(msg) < width: # leaves one character to avoid linefeed + return msg + if len("%s" % current) < width: + return "%s" % current + + # --- adaptive layout algorithm --- + # + # [x] describe the format of the progress bar + # [x] describe min width for each data field + # [x] set priorities for each element + # [x] select elements to be shown + # [x] choose top priority element min_width < avail_width + # [x] lessen avail_width by value if min_width + # [x] exclude element from priority list and repeat + + # 10% [.. ] 10/100 + # pppp bbbbb sssssss + + min_width = { + 'percent': 4, # 100% + 'bar': 3, # [.] + 'size': len("%s" % total)*2 + 3, # 'xxxx / yyyy' + } + priority = ['percent', 'bar', 'size'] + + # select elements to show + selected = [] + avail = width + for field in priority: + if min_width[field] < avail: + selected.append(field) + avail -= min_width[field]+1 # +1 is for separator or for reserved space at + # the end of line to avoid linefeed on Windows + # render + output = '' + for field in selected: + + if field == 'percent': + # fixed size width for percentage + output += ('%s%%' % (100 * current // total)).rjust(min_width['percent']) + elif field == 'bar': # [. ] + # bar takes its min width + all available space + output += bar_thermometer(current, total, min_width['bar']+avail) + elif field == 'size': + # size field has a constant width (min == max) + output += ("%s / %s" % (current, total)).rjust(min_width['size']) + + selected = selected[1:] + if selected: + output += ' ' # add field separator + + return output + +# --/ console helpers + + +__current_size = 0 # global state variable, which exists solely as a + # workaround against Python 3.3.0 regression + # http://bugs.python.org/issue16409 + # fixed in Python 3.3.1 +def callback_progress(blocks, block_size, total_size, bar_function): + """callback function for urlretrieve that is called when connection is + created and when once for each block + + draws adaptive progress bar in terminal/console + + use sys.stdout.write() instead of "print,", because it allows one more + symbol at the line end without linefeed on Windows + + :param blocks: number of blocks transferred so far + :param block_size: in bytes + :param total_size: in bytes, can be -1 if server doesn't return it + :param bar_function: another callback function to visualize progress + """ + global __current_size + + width = min(100, get_console_width()) + + if sys.version_info[:3] == (3, 3, 0): # regression workaround + if blocks == 0: # first call + __current_size = 0 + else: + __current_size += block_size + current_size = __current_size + else: + current_size = min(blocks*block_size, total_size) + progress = bar_function(current_size, total_size, width) + if progress: + sys.stdout.write("\r" + progress) + +class ThrowOnErrorOpener(urllib.FancyURLopener): + def http_error_default(self, url, fp, errcode, errmsg, headers): + raise Exception("%s: %s" % (errcode, errmsg)) + +def download(url, out=None, bar=bar_adaptive): + """High level function, which downloads URL into tmp file in current + directory and then renames it to filename autodetected from either URL + or HTTP headers. + + :param bar: function to track download progress (visualize etc.) + :param out: output filename or directory + :return: filename where URL is downloaded to + """ + names = dict() + names["out"] = out or '' + names["url"] = filename_from_url(url) + # get filename for temp file in current directory + prefix = (names["url"] or names["out"] or ".") + "." + (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=prefix, dir="/tmp") + os.close(fd) + os.unlink(tmpfile) + + # set progress monitoring callback + def callback_charged(blocks, block_size, total_size): + # 'closure' to set bar drawing function in callback + callback_progress(blocks, block_size, total_size, bar_function=bar) + if bar: + callback = callback_charged + else: + callback = None + + (tmpfile, headers) = ThrowOnErrorOpener().retrieve(url, tmpfile, callback) + names["header"] = filename_from_headers(headers) + if os.path.isdir(names["out"]): + filename = names["header"] or names["url"] + filename = names["out"] + "/" + filename + else: + filename = names["out"] or names["header"] or names["url"] + # add numeric ' (x)' suffix if filename already exists + if os.path.exists(filename): + filename = filename_fix_existing(filename) + shutil.move(tmpfile, filename) + + #print headers + return filename + + +usage = """\ +usage: wget.py [options] URL + +options: + -o --output FILE|DIR output filename or directory + -h --help + --version +""" + +if __name__ == "__main__": + if len(sys.argv) < 2 or "-h" in sys.argv or "--help" in sys.argv: + sys.exit(usage) + if "--version" in sys.argv: + sys.exit("wget.py " + __version__) + + from optparse import OptionParser + parser = OptionParser() + parser.add_option("-o", "--output", dest="output") + (options, args) = parser.parse_args() + + url = sys.argv[1] + filename = download(args[0], out=options.output) + + print("") + print("Saved under %s" % filename) + +r""" +features that require more tuits for urlretrieve API +http://www.python.org/doc/2.6/library/urllib.html#urllib.urlretrieve + +[x] autodetect filename from URL +[x] autodetect filename from headers - Content-Disposition + http://greenbytes.de/tech/tc2231/ +[ ] make HEAD request to detect temp filename from Content-Disposition +[ ] process HTTP status codes (i.e. 404 error) + http://ftp.de.debian.org/debian/pool/iso-codes_3.24.2.orig.tar.bz2 +[ ] catch KeyboardInterrupt +[ ] optionally preserve incomplete file +[x] create temp file in current directory +[ ] resume download (broken connection) +[ ] resume download (incomplete file) +[x] show progress indicator + http://mail.python.org/pipermail/tutor/2005-May/038797.html +[x] do not overwrite downloaded file + [x] rename file automatically if exists +[x] optionally specify path for downloaded file + +[ ] options plan + [x] -h, --help, --version (CHAOS speccy) +[ ] clpbar progress bar style +_ 30.0Mb at 3.0 Mbps eta: 0:00:20 30% [===== ] +[ ] test "bar \r" print with \r at the end of line on Windows +[ ] process Python 2.x urllib.ContentTooShortError exception gracefully + (ideally retry and continue download) + + (tmpfile, headers) = urllib.urlretrieve(url, tmpfile, callback_progress) + File "C:\Python27\lib\urllib.py", line 93, in urlretrieve + return _urlopener.retrieve(url, filename, reporthook, data) + File "C:\Python27\lib\urllib.py", line 283, in retrieve + "of %i bytes" % (read, size), result) +urllib.ContentTooShortError: retrieval incomplete: got only 15239952 out of 24807571 bytes + +[ ] find out if urlretrieve may return unicode headers +[ ] test suite for unsafe filenames from url and from headers + +[ ] security checks + [ ] filename_from_url + [ ] filename_from_headers + [ ] MITM redirect from https URL + [ ] https certificate check + [ ] size+hash check helpers + [ ] fail if size is known and mismatch + [ ] fail if hash mismatch +""" From 2ddbcddc4e8af4888c5235e1635f8998b96985d3 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 6 May 2022 14:03:15 +0545 Subject: [PATCH 13/45] user agent added while sending get or head request; --- lambda_fns/extract_docs/app.py | 10 +++++++--- lambda_fns/extract_docs/content_types.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 0a750919..f193f21d 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -45,6 +45,10 @@ sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) +REQ_HEADERS = { + 'user-agent': ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1') +} + class ExtractionStatus(Enum): FAILED = 0 @@ -252,7 +256,7 @@ def get_extracted_text_web_links(link, file_name, mock=False): def handle_urls(url, mock=False): file_name = None - content_type = extract_content_type.get_content_type(url) + content_type = extract_content_type.get_content_type(url, REQ_HEADERS) file_name = EXTRACTED_FILE_NAME @@ -277,7 +281,7 @@ def handle_urls(url, mock=False): s3_file_path = None s3_images_path = None - response = requests.get(url, stream=True) + response = requests.get(url, headers=REQ_HEADERS, stream=True) with tempfile.NamedTemporaryFile(mode='w+b') as temp: for chunk in response.iter_content(chunk_size=128): temp.write(chunk) @@ -305,7 +309,7 @@ def handle_urls(url, mock=False): ext_type = content_type - response = requests.get(url, stream=True) + response = requests.get(url, headers=REQ_HEADERS, stream=True) with tempfile.NamedTemporaryFile(mode='w+b') as temp: for chunk in response.iter_content(chunk_size=128): diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index 7144baaa..7716e55a 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -35,9 +35,9 @@ def __init__(self): self.content_types_xls = ('application/vnd.ms-excel') self.content_types_img = ('image/jpeg', 'image/gif', 'image/png', 'image/svg+xml', 'image/webp', 'image/bmp', 'image/tiff') - def get_content_type(self, url): + def get_content_type(self, url, req_headers): try: - response = requests.head(url) + response = requests.head(url, headers=req_headers) content_type = response.headers['Content-Type'] logging.info(f'The content type of {url} is {content_type}') From 4623d60796d51bcd7bdb15365385de77bfb6a467 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 27 Jul 2022 10:34:53 +0545 Subject: [PATCH 14/45] doc conversion is done by uploading in s3 and triggering lambda with bucket/key payload; --- dev.tfvars | 3 + lambda_fns/extract_docs/app.py | 130 ++++++++++++------ lambda_fns/extract_docs/content_types.py | 2 +- lambda_fns/extract_docs/requirements.txt | 2 +- main.tf | 4 + .../reserved_sqs_lambda_extract_docs/main.tf | 8 +- .../variables.tf | 2 + modules/sqs_lambda_extract_docs/main.tf | 6 +- modules/sqs_lambda_extract_docs/variables.tf | 2 + prod.tfvars | 3 + staging.tfvars | 3 + variables.tf | 2 + 12 files changed, 120 insertions(+), 47 deletions(-) diff --git a/dev.tfvars b/dev.tfvars index 60c204ed..5d671977 100644 --- a/dev.tfvars +++ b/dev.tfvars @@ -19,5 +19,8 @@ docs_extract_fn_image_name = "extractor-tool" # docs convert lambda docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" +# docs convert bucket name +docs_convert_bucket_name = "deep-large-docs-conversion" + # sentry url sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index f193f21d..dfc2f8be 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -9,6 +9,7 @@ from datetime import datetime from enum import Enum import boto3 +from botocore.exceptions import ClientError import tempfile import signal @@ -31,6 +32,7 @@ dest_bucket_name = os.environ.get("DEST_S3_BUCKET") processed_queue_name = os.environ.get("PROCESSED_QUEUE") DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") +DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") domain_name = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") @@ -79,6 +81,39 @@ def store_text_s3(data, bucket_name, key): Key=key ) +def upload_file_to_s3( + url, + key="temporaryfile.pdf", + bucketname="deep-large-docs-conversion" +): + try: + session = boto3.Session() + s3_resource = session.resource("s3") + + r = requests.get(url, stream=True) + if r.status_code == 200: + bucket = s3_resource.Bucket(bucketname) + bucket.upload_fileobj(r.raw, key) + logging.info(f"The file is uploaded to the S3 bucket {bucketname}") + else: + return False + except Exception as e: + logging.error(str(e)) + return False + return True + +def download_file(filename_s3, bucketname, filename_local): + try: + s3_client.download_file( + bucketname, + filename_s3, + filename_local + ) + logging.info("The file is downloaded in the lambda /tmp directory.") + except ClientError as e: + return False + return True + def get_words_count(text): if text: @@ -151,14 +186,14 @@ def get_extracted_content_links(file_path, file_name, mock): document = TextFromFile(stream=binary, ext="pdf") entries, images = document.serial_extract_text(output_format="list") except Exception as e: - logging.error(f"Extraction failed: {e}") + logging.error(f"Extraction failed: {str(e)}") return None, None, -1, -1 entries_list = [item for sublist in entries for item in sublist] extracted_text = "\n".join(entries_list) - + extracted_text = extracted_text.replace("\x00", "") # remove null chars - + extracted_text = extracted_text.encode('utf-8', 'ignore').decode('utf-8') total_pages = len(entries) total_words_count = get_words_count(extracted_text) @@ -286,11 +321,15 @@ def handle_urls(url, mock=False): for chunk in response.iter_content(chunk_size=128): temp.write(chunk) temp.seek(0) + try: s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( temp.name, file_name, mock ) - extraction_status = ExtractionStatus.SUCCESS.value + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value + else: + extraction_status = ExtractionStatus.FAILED.value except Exception: s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value @@ -299,7 +338,10 @@ def handle_urls(url, mock=False): s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_text_web_links( url, file_name, mock ) - extraction_status = ExtractionStatus.SUCCESS.value + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value + else: + extraction_status = ExtractionStatus.FAILED.value except Exception: s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value @@ -308,48 +350,51 @@ def handle_urls(url, mock=False): content_type == UrlTypes.PPTX.value or content_type == UrlTypes.PPT.value: ext_type = content_type + tmp_filename = f"{uuid.uuid4().hex}.{ext_type}" + flag = False + if upload_file_to_s3(url, key=tmp_filename, bucketname=DOCS_CONVERSION_BUCKET_NAME): + payload = json.dumps({ + "file": tmp_filename, + "bucket": DOCS_CONVERSION_BUCKET_NAME, + "ext": ext_type, + "fromS3": 1 + }) + + docs_conversion_lambda_response = lambda_client.invoke( + FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, + InvocationType="RequestResponse", + Payload=payload + ) + docs_conversion_lambda_response_json = json.loads( + docs_conversion_lambda_response["Payload"].read().decode("utf-8") + ) - response = requests.get(url, headers=REQ_HEADERS, stream=True) + if "statusCode" in docs_conversion_lambda_response_json and \ + docs_conversion_lambda_response_json["statusCode"] == 200: + bucket_name = docs_conversion_lambda_response_json["bucket"] + file_path = docs_conversion_lambda_response_json["file"] + filename = file_path.split("/")[-1] - with tempfile.NamedTemporaryFile(mode='w+b') as temp: - for chunk in response.iter_content(chunk_size=128): - temp.write(chunk) - temp.seek(0) - try: - with open(pathlib.Path(temp.name), "rb") as f: - binary = base64.b64encode(f.read()) - - payload = json.dumps({ - "file": binary.decode(), - "ext": ext_type - }) - - response_docx = lambda_client.invoke( - FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, - InvocationType="RequestResponse", - Payload=payload + if download_file(file_path, bucket_name, f"/tmp/{filename}"): + s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( + f"/tmp/{filename}", file_name, mock ) - resp_docx_json = json.loads(response_docx["Payload"].read().decode("utf-8")) - - if resp_docx_json["statusCode"] == 200: - data = resp_docx_json["file"] - data_b64 = base64.b64decode(data) - - with tempfile.NamedTemporaryFile(mode="w+b") as tempf: - tempf.write(data_b64) - tempf.seek(0) - - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( - tempf.name, file_name, mock - ) - extraction_status = ExtractionStatus.SUCCESS.value + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value else: - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value - except Exception as e: - logging.error(f"Exception occurred {e}") - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value + else: + flag = True + else: + logging.error(f"Error occurred during file conversion. {docs_conversion_lambda_response_json['error']}") + flag = True + else: + logging.warn("Could not upload the file to s3.") + flag = True + + if flag: + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value elif content_type == UrlTypes.IMG.value: logging.warn("Text extraction from Images is not available.") s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 @@ -359,6 +404,7 @@ def handle_urls(url, mock=False): logging.info(f"The extracted file path is {s3_file_path}") logging.info(f"The extracted image path is {s3_images_path}") + logging.info(f"The status of the extraction is {str(extraction_status)}") return s3_file_path, s3_images_path, str(total_pages), str(total_words_count), str(extraction_status) diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index 7716e55a..bacc8682 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -25,7 +25,7 @@ class UrlTypes(str, Enum): class ExtractContentType: def __init__(self): self.content_types_pdf = ('application/pdf', 'pdf') - self.content_types_html = ('text/html', 'text/html; charset=utf-8', + self.content_types_html = ('text/html', 'text/html; charset=utf-8', 'text/html;charset=UTF-8', 'text/html; charset=UTF-8', 'text/html;charset=utf-8', 'text/plain') self.content_types_docx = ('application/vnd.openxmlformats-officedocument.wordprocessingml.document') self.content_types_doc = ('application/msword') diff --git a/lambda_fns/extract_docs/requirements.txt b/lambda_fns/extract_docs/requirements.txt index 4eb5689c..b2708ffc 100644 --- a/lambda_fns/extract_docs/requirements.txt +++ b/lambda_fns/extract_docs/requirements.txt @@ -10,5 +10,5 @@ six==1.16.0 urllib3==1.26.6 Pillow==8.3.2 lxml==4.7.1 -sentry-sdk==1.5.8 deep-parser @ git+https://github.com/the-deep/deepex +sentry-sdk==1.5.8 diff --git a/main.tf b/main.tf index 00b5d9f7..b3cb3b77 100644 --- a/main.tf +++ b/main.tf @@ -43,6 +43,8 @@ module "sqs_lambda_module" { reserved_input_queue_id = "${module.reserved_sqs_lambda_module.reserved_input_queue_id}" reserved_input_queue_arn = "${module.reserved_sqs_lambda_module.reserved_input_queue_arn}" + docs_convert_bucket_name = var.docs_convert_bucket_name + environment = var.environment sentry_url = var.sentry_url @@ -58,6 +60,8 @@ module "reserved_sqs_lambda_module" { docs_convert_lambda_fn_name = var.docs_convert_lambda_fn_name + docs_convert_bucket_name = var.docs_convert_bucket_name + environment = var.environment sentry_url = var.sentry_url diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index 62fc6f7e..dde5ed7a 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -101,7 +101,7 @@ module "reserved_extract_docs_fn" { pip_requirements = "${path.module}/../../lambda_fns/extract_docs/requirements.txt" }] - memory_size = 2048 + memory_size = 4096 attach_policy_json = true policy_json = jsonencode({ @@ -118,13 +118,16 @@ module "reserved_extract_docs_fn" { "sqs:GetQueueUrl", "sqs:ListQueues", "sqs:SendMessageBatch", - "s3:PutObject" + "s3:PutObject", + "s3:GetObject" ], "Resource": [ aws_sqs_queue.reserved_input_queue.arn, aws_sqs_queue.reserved_processed_queue.arn, "${var.processed_docs_bucket_arn}", "${var.processed_docs_bucket_arn}/*", + "arn:aws:s3:::${var.docs_convert_bucket_name}", + "arn:aws:s3:::${var.docs_convert_bucket_name}/*", "arn:aws:lambda:us-east-1:${data.aws_caller_identity.reserved_current_user.account_id}:function:${var.docs_convert_lambda_fn_name}" ] } @@ -140,6 +143,7 @@ module "reserved_extract_docs_fn" { DEST_S3_BUCKET = "${var.processed_docs_bucket}" PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id DOCS_CONVERT_LAMBDA_FN_NAME = "${var.docs_convert_lambda_fn_name}" + DOCS_CONVERSION_BUCKET_NAME = "${var.docs_convert_bucket_name}" ENVIRONMENT = "${var.environment}" SENTRY_URL = "${var.sentry_url}" } diff --git a/modules/reserved_sqs_lambda_extract_docs/variables.tf b/modules/reserved_sqs_lambda_extract_docs/variables.tf index 2b183b0b..170aa563 100644 --- a/modules/reserved_sqs_lambda_extract_docs/variables.tf +++ b/modules/reserved_sqs_lambda_extract_docs/variables.tf @@ -15,4 +15,6 @@ variable "docs_extract_fn_image_name" {} variable "docs_convert_lambda_fn_name" {} +variable docs_convert_bucket_name {} + variable sentry_url {} \ No newline at end of file diff --git a/modules/sqs_lambda_extract_docs/main.tf b/modules/sqs_lambda_extract_docs/main.tf index 73c8b620..80a42090 100644 --- a/modules/sqs_lambda_extract_docs/main.tf +++ b/modules/sqs_lambda_extract_docs/main.tf @@ -126,13 +126,16 @@ module "extract_docs_fn" { "sqs:GetQueueUrl", "sqs:ListQueues", "sqs:SendMessageBatch", - "s3:PutObject" + "s3:PutObject", + "s3:GetObject" ], "Resource": [ aws_sqs_queue.input_queue.arn, aws_sqs_queue.processed_queue.arn, "${var.processed_docs_bucket_arn}", "${var.processed_docs_bucket_arn}/*", + "arn:aws:s3:::${var.docs_convert_bucket_name}", + "arn:aws:s3:::${var.docs_convert_bucket_name}/*", "arn:aws:lambda:us-east-1:${data.aws_caller_identity.current_user.account_id}:function:${var.docs_convert_lambda_fn_name}" ] } @@ -145,6 +148,7 @@ module "extract_docs_fn" { DEST_S3_BUCKET = "${var.processed_docs_bucket}" PROCESSED_QUEUE = aws_sqs_queue.processed_queue.id DOCS_CONVERT_LAMBDA_FN_NAME = "${var.docs_convert_lambda_fn_name}" + DOCS_CONVERSION_BUCKET_NAME = "${var.docs_convert_bucket_name}" ENVIRONMENT = "${var.environment}" SENTRY_URL = "${var.sentry_url}" } diff --git a/modules/sqs_lambda_extract_docs/variables.tf b/modules/sqs_lambda_extract_docs/variables.tf index 7bf2f820..c6a84fe2 100644 --- a/modules/sqs_lambda_extract_docs/variables.tf +++ b/modules/sqs_lambda_extract_docs/variables.tf @@ -15,6 +15,8 @@ variable "docs_extract_fn_image_name" {} variable "docs_convert_lambda_fn_name" {} +variable docs_convert_bucket_name {} + variable reserved_input_queue_id {} variable reserved_input_queue_arn {} diff --git a/prod.tfvars b/prod.tfvars index 02be5089..11accef3 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -19,5 +19,8 @@ docs_extract_fn_image_name = "extract-tool" # docs convert lambda docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" +# docs convert bucket name +docs_convert_bucket_name = "deep-large-docs-conversion" + # sentry url sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file diff --git a/staging.tfvars b/staging.tfvars index 9f7b840b..19d90882 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -19,5 +19,8 @@ docs_extract_fn_image_name = "extract-tool" # docs convert lambda docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" +# docs convert bucket name +docs_convert_bucket_name = "deep-large-docs-conversion" + # sentry url sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file diff --git a/variables.tf b/variables.tf index fadc34df..06813dd3 100644 --- a/variables.tf +++ b/variables.tf @@ -22,4 +22,6 @@ variable docs_extract_fn_image_name {} variable docs_convert_lambda_fn_name {} +variable docs_convert_bucket_name {} + variable sentry_url {} \ No newline at end of file From db7b4927a4207b20257ae3227d9420cdc80fd16f Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Mon, 1 Aug 2022 16:47:25 +0545 Subject: [PATCH 15/45] deploy deepex parser in the ecs; --- lambda_fns/extract_docs/app.py | 212 +++++++++--------- main.tf | 36 +++ modules/ecs/cloudwatch.tf | 13 ++ modules/ecs/ecs.tf | 46 ++++ modules/ecs/iam_roles.tf | 77 +++++++ modules/ecs/main.tf | 0 modules/ecs/output.tf | 15 ++ modules/ecs/templates/ecr_image/image.json | 17 ++ modules/ecs/variables.tf | 33 +++ modules/ecs/vpc.tf | 88 ++++++++ .../reserved_sqs_lambda_extract_docs/main.tf | 21 +- .../variables.tf | 8 + modules/sqs_lambda_extract_docs/main.tf | 19 +- modules/sqs_lambda_extract_docs/variables.tf | 8 + staging.tfvars | 15 +- variables.tf | 16 +- 16 files changed, 515 insertions(+), 109 deletions(-) create mode 100644 modules/ecs/cloudwatch.tf create mode 100644 modules/ecs/ecs.tf create mode 100644 modules/ecs/iam_roles.tf create mode 100644 modules/ecs/main.tf create mode 100644 modules/ecs/output.tf create mode 100644 modules/ecs/templates/ecr_image/image.json create mode 100644 modules/ecs/variables.tf create mode 100644 modules/ecs/vpc.tf diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index dfc2f8be..7681545d 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -11,7 +11,6 @@ import boto3 from botocore.exceptions import ClientError import tempfile -import signal import sentry_sdk @@ -28,20 +27,20 @@ EXTRACTED_FILE_NAME = "extract_text.txt" DEFAULT_AWS_REGION = "us-east-1" -aws_region = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) -dest_bucket_name = os.environ.get("DEST_S3_BUCKET") -processed_queue_name = os.environ.get("PROCESSED_QUEUE") +AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) +DEST_BUCKET_NAME = os.environ.get("DEST_S3_BUCKET") +PROCESSED_QUEUE_NAME = os.environ.get("PROCESSED_QUEUE") DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") -domain_name = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") +DOMAIN_NAME = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") SENTRY_URL = os.environ.get("SENTRY_URL") ENVIRONMENT = os.environ.get("ENVIRONMENT") -s3_client = boto3.client('s3', region_name=aws_region) -sqs_client = boto3.client('sqs', region_name=aws_region) -lambda_client = boto3.client('lambda', region_name="us-east-1") +s3_client = boto3.client('s3', region_name=AWS_REGION) +sqs_client = boto3.client('sqs', region_name=AWS_REGION) +lambda_client = boto3.client('lambda', region_name=AWS_REGION) extract_content_type = ExtractContentType() @@ -57,10 +56,6 @@ class ExtractionStatus(Enum): SUCCESS = 1 -def timeout_handler(_signal, _frame): - raise Exception("Lambda timeout occurred.") - - def extract_path(filepath): """ Extracts bucket and key names from the s3 URI @@ -123,61 +118,6 @@ def get_words_count(text): return 0 -def send_message2sqs( - client_id, - url, - callback_url, - s3_text_path, - s3_images_path, - total_pages, - total_words_count, - extraction_status -): - message_attributes = {} - message_attributes['url'] = { - 'DataType': 'String', - 'StringValue': url - } - message_attributes['extraction_status'] = { - 'DataType': 'Number', - 'StringValue': extraction_status if s3_text_path else "0" - } - - if s3_text_path: - message_attributes['s3_text_path'] = { - 'DataType': 'String', - 'StringValue': s3_text_path - } - - if s3_images_path: - message_attributes['s3_images_path'] = { - 'DataType': 'String', - 'StringValue': s3_images_path - } - - message_attributes['callback_url'] = { - 'DataType': 'String', - 'StringValue': callback_url - } - message_attributes['total_pages'] = { - 'DataType': 'Number', - 'StringValue': total_pages - } - message_attributes['total_words_count'] = { - 'DataType': 'Number', - 'StringValue': total_words_count - } - if processed_queue_name: - sqs_client.send_message( - QueueUrl=processed_queue_name, - MessageBody=client_id, - DelaySeconds=0, - MessageAttributes=message_attributes - ) - else: - logging.error("Message not sent to the processed SQS.") - - def get_extracted_content_links(file_path, file_name, mock): try: with open(pathlib.Path(file_path), "rb") as f: @@ -212,7 +152,7 @@ def get_extracted_content_links(file_path, file_name, mock): dir_images_path.mkdir() if not dir_images_path.exists() else None images.save_images(directory_path=dir_images_path) - s3_file_path = f'{domain_name}{str(dir_path)}/{file_name}' + s3_file_path = f'{DOMAIN_NAME}{str(dir_path)}/{file_name}' s3_images_path = [] # Note: commented for now @@ -220,12 +160,12 @@ def get_extracted_content_links(file_path, file_name, mock): # for f in files: # full_path = os.path.join(subdir, f) # with open(full_path, 'rb') as data: - # s3_images_path.append(f"{domain_name}{str(dir_images_path)}/{f}") + # s3_images_path.append(f"{DOMAIN_NAME}{str(dir_images_path)}/{f}") return s3_file_path, s3_images_path, total_pages, total_words_count else: store_text_s3( extracted_text, - dest_bucket_name, + DEST_BUCKET_NAME, f"{str(s3_path_prefix)}/{file_name}" ) @@ -240,12 +180,12 @@ def get_extracted_content_links(file_path, file_name, mock): # with open(full_path, 'rb') as data: # store_text_s3( # data, - # dest_bucket_name, + # DEST_BUCKET_NAME, # f"{str(s3_path_prefix)}/images/{f}" # ) - s3_file_path = f"s3://{dest_bucket_name}/{str(s3_path_prefix)}/{file_name}" - s3_images_path = f"s3://{dest_bucket_name}/{str(s3_path_prefix)}/images" + s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" + s3_images_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/images" return s3_file_path, s3_images_path, total_pages, total_words_count @@ -277,20 +217,18 @@ def get_extracted_text_web_links(link, file_name, mock=False): # os.makedirs(f'{dir_path}/images') with open(dir_path / file_name, 'w') as f: f.write(extracted_text) - s3_file_path = f'{domain_name}{str(dir_path)}/{file_name}' + s3_file_path = f'{DOMAIN_NAME}{str(dir_path)}/{file_name}' else: store_text_s3( extracted_text, - dest_bucket_name, + DEST_BUCKET_NAME, f"{str(s3_path_prefix)}/{file_name}" ) - s3_file_path = f"s3://{dest_bucket_name}/{str(s3_path_prefix)}/{file_name}" + s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" return s3_file_path, None, total_pages, total_words_count # No images extraction (lib doesn't support?) def handle_urls(url, mock=False): - file_name = None - content_type = extract_content_type.get_content_type(url, REQ_HEADERS) file_name = EXTRACTED_FILE_NAME @@ -409,6 +347,90 @@ def handle_urls(url, mock=False): return s3_file_path, s3_images_path, str(total_pages), str(total_words_count), str(extraction_status) +def run_fargate_task( + client_id, + url, + callback_url +): + VPC_PRIVATE_SUBNET = os.environ.get("VPC_PRIVATE_SUBNET") + ECS_CLUSTER_ID = os.environ.get("ECS_CLUSTER_ID") + ECS_TASK_DEFINITION = os.environ.get("ECS_TASK_DEFINITION") + ECS_CONTAINER_NAME = os.environ.get("ECS_CONTAINER_NAME") + + ecs_client = boto3.client('ecs', region_name=AWS_REGION) + + response = ecs_client.run_task( + cluster=ECS_CLUSTER_ID, + launchType='FARGATE', + taskDefinition=ECS_TASK_DEFINITION, + count = 1, + platformVersion='LATEST', + networkConfiguration={ + 'awsvpcConfiguration': { + 'subnets': [ + VPC_PRIVATE_SUBNET + ], + 'assignPublicIp': 'DISABLED' + } + }, + overrides={ + 'containerOverrides': [ + { + 'name': f"{ECS_CONTAINER_NAME}-{ENVIRONMENT}", + 'command': [ + 'python', + 'app.py' + ], + "environment": [ + { + "name": "CLIENT_ID", + "value": client_id + }, + { + "name": "URL", + "value": url + }, + { + "name": "CALLBACK_URL", + "value": callback_url + }, + { + "name": "AWS_REGION", + "value": AWS_REGION + }, + { + "name": "ENVIRONMENT", + "value": ENVIRONMENT + }, + { + "name": "DEST_S3_BUCKET", + "value": DEST_BUCKET_NAME + }, + { + "name": "DOCS_CONVERSION_BUCKET_NAME", + "value": DOCS_CONVERSION_BUCKET_NAME + }, + { + "name": "DOCS_CONVERT_LAMBDA_FN_NAME", + "value": DOCS_CONVERT_LAMBDA_FN_NAME + }, + { + "name": "PROCESSED_QUEUE", + "value": PROCESSED_QUEUE_NAME + }, + { + "name": "SENTRY_URL", + "value": SENTRY_URL + } + ] + }, + ], + }, + ) + + return str(response) #handle the response status + + def process_docs(event, context): logging.debug(f"The event output is {event}") @@ -426,37 +448,23 @@ def process_docs(event, context): } else: try: - signal.alarm(int(context.get_remaining_time_in_millis() / 1000) - 120) records = event['Records'] for record in records: client_id = record['body'] url = record['messageAttributes']['url']['stringValue'] callback_url = record['messageAttributes']['callback_url']['stringValue'] - logging.info(f"Processing {url}") - - s3_text_path, s3_images_path, total_pages, total_words_count, extraction_status = handle_urls(url) - - sqs_message = { - 'client_id': client_id, - 'url': url, - 'callback_url': callback_url, - 's3_text_path': s3_text_path, - 's3_images_path': s3_images_path, - 'total_pages': total_pages, - 'total_words_count': total_words_count, - 'extraction_status': extraction_status - } - send_message2sqs(**sqs_message) - except Exception as e: - logging.error(e, exc_info=True) - signal.alarm(0) + logging.info(f"Processing {url} in the ecs fargate.") + run_fargate_task(client_id, url, callback_url) + except Exception as e: + logging.error(e, exc_info=True) + return { + "statusCode": 500, + "body": f"Error occurred: {str(e)}" + } return { - 'statusCode': 200, - 'body': 'Message processed successfully.' + "statusCode": 200, + "body": "Message processed successfully." } - - -signal.signal(signal.SIGALRM, timeout_handler) diff --git a/main.tf b/main.tf index b3cb3b77..fad4007f 100644 --- a/main.tf +++ b/main.tf @@ -45,6 +45,14 @@ module "sqs_lambda_module" { docs_convert_bucket_name = var.docs_convert_bucket_name + vpc_private_subnet = "${module.ecs_module.private_subnet_id}" + + ecs_cluster_id = "${module.ecs_module.ecs_cluster_id}" + + ecs_task_definition = "${module.ecs_module.ecs_task_definition}" + + ecs_container_name = "${module.ecs_module.ecs_container_name}" + environment = var.environment sentry_url = var.sentry_url @@ -62,6 +70,14 @@ module "reserved_sqs_lambda_module" { docs_convert_bucket_name = var.docs_convert_bucket_name + vpc_private_subnet = "${module.ecs_module.private_subnet_id}" + + ecs_cluster_id = "${module.ecs_module.ecs_cluster_id}" + + ecs_task_definition = "${module.ecs_module.ecs_task_definition}" + + ecs_container_name = "${module.ecs_module.ecs_container_name}" + environment = var.environment sentry_url = var.sentry_url @@ -133,4 +149,24 @@ module "s3_module" { source = "./modules/s3" environment = var.environment +} + +module "ecs_module" { + source = "./modules/ecs" + + environment = var.environment + aws_region = var.aws_region + + # VPC + az_count = var.az_count + cidr_block = var.cidr_block + + # ECS role + ecs_task_execution_role = var.ecs_task_execution_role + + # ECS + fargate_cpu = var.fargate_cpu + fargate_memory = var.fargate_memory + app_count = var.app_count + app_image = var.app_image } \ No newline at end of file diff --git a/modules/ecs/cloudwatch.tf b/modules/ecs/cloudwatch.tf new file mode 100644 index 00000000..193a14a5 --- /dev/null +++ b/modules/ecs/cloudwatch.tf @@ -0,0 +1,13 @@ +resource "aws_cloudwatch_log_group" "testapp_log_group" { + name = "/ecs/deepex-parser-${var.environment}" + retention_in_days = 30 + + tags = { + Name = "cw-log-group-deepex-parser-${var.environment}" + } +} + +resource "aws_cloudwatch_log_stream" "myapp_log_stream" { + name = "log-stream-deepex-parser-${var.environment}" + log_group_name = aws_cloudwatch_log_group.testapp_log_group.name +} \ No newline at end of file diff --git a/modules/ecs/ecs.tf b/modules/ecs/ecs.tf new file mode 100644 index 00000000..f0875274 --- /dev/null +++ b/modules/ecs/ecs.tf @@ -0,0 +1,46 @@ +resource "aws_ecs_cluster" "cluster" { + name = "${var.ecs_cluster_name}-${var.environment}" +} + +data "template_file" "config" { + template = file("./modules/ecs/templates/ecr_image/image.json") + + vars = { + container_name = var.ecs_container + app_image = var.app_image + fargate_cpu = var.fargate_cpu + fargate_memory = var.fargate_memory + aws_region = var.aws_region + environment = var.environment + } +} + +resource "aws_ecs_task_definition" "task-def" { + family = "${var.ecs_task_definition}-${var.environment}" + task_role_arn = aws_iam_role.ecs-custom-tasks.arn + execution_role_arn = aws_iam_role.ecs_task_execution_role.arn + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + cpu = var.fargate_cpu + memory = var.fargate_memory + container_definitions = data.template_file.config.rendered +} + +resource "aws_ecs_service" "service" { + name = "${var.ecs_service}-${var.environment}" + cluster = aws_ecs_cluster.cluster.id + task_definition = aws_ecs_task_definition.task-def.arn + desired_count = var.app_count + launch_type = "FARGATE" + + network_configuration { + #security_groups = [aws_security_group.ecs_sg.id] + subnets = aws_subnet.private.*.id + assign_public_ip = false + } + + depends_on = [ + aws_iam_role_policy_attachment.ecs_task_execution_role, + aws_iam_role_policy_attachment.ecs-tasks + ] +} \ No newline at end of file diff --git a/modules/ecs/iam_roles.tf b/modules/ecs/iam_roles.tf new file mode 100644 index 00000000..206cef9b --- /dev/null +++ b/modules/ecs/iam_roles.tf @@ -0,0 +1,77 @@ +data "aws_iam_policy_document" "ecs_task_execution_role" { + version = "2012-10-17" + statement { + sid = "" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "ecs_task_execution_role" { + name = "${var.ecs_task_execution_role}-${var.environment}" + assume_role_policy = data.aws_iam_policy_document.ecs_task_execution_role.json +} + +resource "aws_iam_role_policy_attachment" "ecs_task_execution_role" { + role = aws_iam_role.ecs_task_execution_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +resource "aws_iam_role" "ecs-custom-tasks" { + name = "ecs-custom-tasks" + + assume_role_policy = < Date: Thu, 4 Aug 2022 11:19:24 +0545 Subject: [PATCH 16/45] deepex parser dockerfile; --- lambda_fns/extract_docs/Dockerfile | 28 + lambda_fns/extract_docs/deepex_ecs/app.py | 406 +++++++++++++ lambda_fns/extract_docs/poetry.lock | 705 ++++++++++++++++++++++ lambda_fns/extract_docs/pyproject.toml | 27 + 4 files changed, 1166 insertions(+) create mode 100644 lambda_fns/extract_docs/Dockerfile create mode 100644 lambda_fns/extract_docs/deepex_ecs/app.py create mode 100644 lambda_fns/extract_docs/poetry.lock create mode 100644 lambda_fns/extract_docs/pyproject.toml diff --git a/lambda_fns/extract_docs/Dockerfile b/lambda_fns/extract_docs/Dockerfile new file mode 100644 index 00000000..f661b7ab --- /dev/null +++ b/lambda_fns/extract_docs/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1.2 + +FROM python:3.10-slim-buster + +LABEL maintainer="Deep Dev dev@thedeep.com" + +WORKDIR /code + +COPY pyproject.toml poetry.lock /code/ + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + # Build required packages + git gcc curl \ + # Upgrade pip and install python packages for code + && cd /code/ \ + && pip install --upgrade --no-cache-dir pip poetry \ + && poetry --version \ + # Configure to use system instead of virtualenvs + && poetry config virtualenvs.create false \ + && poetry install --no-root \ + # Clean-up + && pip uninstall -y poetry virtualenv-clone virtualenv \ + && apt-get remove -y gcc \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +COPY deepex_ecs/app.py content_types.py wget.py /code/ diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py new file mode 100644 index 00000000..708ac659 --- /dev/null +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -0,0 +1,406 @@ +import os +import requests +import uuid +import json +import base64 +import logging +import re +import pathlib +from datetime import datetime +from enum import Enum +import boto3 +from botocore.exceptions import ClientError +import tempfile + +import sentry_sdk + +from deep_parser import TextFromFile +from deep_parser import TextFromWeb + +try: + from content_types import ExtractContentType, UrlTypes +except ImportError: + from .content_types import ExtractContentType, UrlTypes + +logging.getLogger().setLevel(logging.INFO) + +EXTRACTED_FILE_NAME = "extract_text.txt" +DEFAULT_AWS_REGION = "us-east-1" + +AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) +DEST_BUCKET_NAME = os.environ.get("DEST_S3_BUCKET") +PROCESSED_QUEUE_NAME = os.environ.get("PROCESSED_QUEUE") +DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") +DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") + +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + +CLIENT_ID = os.environ.get("CLIENT_ID") +URL = os.environ.get("URL") +CALLBACK_URL = os.environ.get("CALLBACK_URL") + +s3_client = boto3.client('s3', region_name=AWS_REGION) +sqs_client = boto3.client('sqs', region_name=AWS_REGION) +lambda_client = boto3.client('lambda', region_name=AWS_REGION) + +extract_content_type = ExtractContentType() + +sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) + +REQ_HEADERS = { + 'user-agent': ('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1') +} + + +class ExtractionStatus(Enum): + FAILED = 0 + SUCCESS = 1 + + +def extract_path(filepath): + """ + Extracts bucket and key names from the s3 URI + """ + if filepath is not None: + filepath_split = pathlib.Path(filepath).parts + bucket_name = filepath_split[1] + file_key_path = str(pathlib.Path(*filepath_split[2:])) + file_name = filepath_split[-1] + return bucket_name, file_key_path, file_name + return None, None, None + + +def store_text_s3(data, bucket_name, key): + s3_client.put_object( + Body=data, + Bucket=bucket_name, + Key=key + ) + + +def upload_file_to_s3( + url, + key="temporaryfile.pdf", + bucketname="deep-large-docs-conversion" +): + try: + session = boto3.Session() + s3_resource = session.resource("s3") + + r = requests.get(url, stream=True) + if r.status_code == 200: + bucket = s3_resource.Bucket(bucketname) + bucket.upload_fileobj(r.raw, key) + logging.info(f"The file is uploaded to the S3 bucket {bucketname}") + else: + return False + except Exception as e: + logging.error(str(e)) + return False + return True + + +def download_file(filename_s3, bucketname, filename_local): + try: + s3_client.download_file( + bucketname, + filename_s3, + filename_local + ) + logging.info("The file is downloaded in the lambda /tmp directory.") + except ClientError as e: + logging.error(f"Client error occurred. {str(e)}") + return False + return True + + +def get_words_count(text): + if text: + w = re.sub(r'[^\w\s]', '', text) + w = re.sub(r'_', '', w) + return len(w.split()) + return 0 + + +def send_message2sqs( + client_id, + url, + callback_url, + s3_text_path, + s3_images_path, + total_pages, + total_words_count, + extraction_status +): + message_attributes = {} + message_attributes['url'] = { + 'DataType': 'String', + 'StringValue': url + } + message_attributes['extraction_status'] = { + 'DataType': 'Number', + 'StringValue': extraction_status if s3_text_path else "0" + } + + if s3_text_path: + message_attributes['s3_text_path'] = { + 'DataType': 'String', + 'StringValue': s3_text_path + } + + if s3_images_path: + message_attributes['s3_images_path'] = { + 'DataType': 'String', + 'StringValue': s3_images_path + } + + message_attributes['callback_url'] = { + 'DataType': 'String', + 'StringValue': callback_url + } + message_attributes['total_pages'] = { + 'DataType': 'Number', + 'StringValue': total_pages + } + message_attributes['total_words_count'] = { + 'DataType': 'Number', + 'StringValue': total_words_count + } + if PROCESSED_QUEUE_NAME: + sqs_client.send_message( + QueueUrl=PROCESSED_QUEUE_NAME, + MessageBody=client_id, + DelaySeconds=0, + MessageAttributes=message_attributes + ) + else: + logging.error("Message not sent to the processed SQS.") + + +def get_extracted_content_links(file_path, file_name): + try: + with open(pathlib.Path(file_path), "rb") as f: + binary = base64.b64encode(f.read()) + + document = TextFromFile(stream=binary, ext="pdf") + entries, images = document.serial_extract_text(output_format="list") + except Exception as e: + logging.error(f"Extraction failed: {str(e)}") + return None, None, -1, -1 + + entries_list = [item for sublist in entries for item in sublist] + extracted_text = "\n".join(entries_list) + + extracted_text = extracted_text.replace("\x00", "") # remove null chars + extracted_text = extracted_text.encode('utf-8', 'ignore').decode('utf-8') + total_pages = len(entries) + total_words_count = get_words_count(extracted_text) + + date_today = str(datetime.now().date()) + + dir_name = str(uuid.uuid4()) + + s3_path_prefix = pathlib.Path(date_today, dir_name) + + store_text_s3( + extracted_text, + DEST_BUCKET_NAME, + f"{str(s3_path_prefix)}/{file_name}" + ) + + local_temp_directory = pathlib.Path('/tmp', file_name) + local_temp_directory.mkdir(parents=True) if not local_temp_directory.exists() else None + # Note: commented for now + # images.save_images(directory_path=local_temp_directory) + + # for subdir, dirs, files in os.walk(local_temp_directory): + # for f in files: + # full_path = os.path.join(subdir, f) + # with open(full_path, 'rb') as data: + # store_text_s3( + # data, + # DEST_BUCKET_NAME, + # f"{str(s3_path_prefix)}/images/{f}" + # ) + + s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" + s3_images_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/images" + + return s3_file_path, s3_images_path, total_pages, total_words_count + + +def get_extracted_text_web_links(link, file_name): + try: + web_text = TextFromWeb(url=link) + entries = web_text.extract_text(output_format="list") + except Exception as e: + logging.error(f"Extraction from website failed {e}") + return None, None, -1, -1 + + entries_list = [item for sublist in entries for item in sublist] + extracted_text = "\n".join(entries_list) + + extracted_text = extracted_text.replace("\x00", "") # remove null chars + + total_pages = 1 + total_words_count = get_words_count(extracted_text) + + date_today = str(datetime.now().date()) + + dir_name = uuid.uuid4().hex + + s3_path_prefix = pathlib.Path(date_today, dir_name) + + store_text_s3( + extracted_text, + DEST_BUCKET_NAME, + f"{str(s3_path_prefix)}/{file_name}" + ) + s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" + return s3_file_path, None, total_pages, total_words_count # No images extraction (lib doesn't support?) + + +def handle_urls(url): + content_type = extract_content_type.get_content_type(url, REQ_HEADERS) + + file_name = EXTRACTED_FILE_NAME + + if url.startswith("s3"): + bucket_name, file_path, file_name = extract_path(url) + local_file_path = f"/tmp/{file_name}" + s3_client.download_file( + bucket_name, + file_path, + local_file_path + ) + try: + s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( + local_file_path, file_name + ) + extraction_status = ExtractionStatus.SUCCESS.value + except Exception as e: + logging.error(e, exc_info=True) + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + + elif content_type == UrlTypes.PDF.value: # assume it is http/https pdf weblink + s3_file_path = None + s3_images_path = None + + response = requests.get(url, headers=REQ_HEADERS, stream=True) + with tempfile.NamedTemporaryFile(mode='w+b') as temp: + for chunk in response.iter_content(chunk_size=128): + temp.write(chunk) + temp.seek(0) + + try: + s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( + temp.name, file_name + ) + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value + else: + extraction_status = ExtractionStatus.FAILED.value + except Exception as e: + logging.error(f"Error occurred during text extraction. {str(e)}", exc_info=True) + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + elif content_type == UrlTypes.HTML.value: # assume it is a static webpage + try: + s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_text_web_links( + url, file_name + ) + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value + else: + extraction_status = ExtractionStatus.FAILED.value + except Exception: + logging.error(f"Error occurred during text extraction. {str(e)}", exc_info=True) + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + elif content_type == UrlTypes.DOCX.value or content_type == UrlTypes.MSWORD.value or \ + content_type == UrlTypes.XLSX.value or content_type == UrlTypes.XLS.value or \ + content_type == UrlTypes.PPTX.value or content_type == UrlTypes.PPT.value: + + ext_type = content_type + tmp_filename = f"{uuid.uuid4().hex}.{ext_type}" + flag = False + if upload_file_to_s3(url, key=tmp_filename, bucketname=DOCS_CONVERSION_BUCKET_NAME): + payload = json.dumps({ + "file": tmp_filename, + "bucket": DOCS_CONVERSION_BUCKET_NAME, + "ext": ext_type, + "fromS3": 1 + }) + + docs_conversion_lambda_response = lambda_client.invoke( + FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, + InvocationType="RequestResponse", + Payload=payload + ) + docs_conversion_lambda_response_json = json.loads( + docs_conversion_lambda_response["Payload"].read().decode("utf-8") + ) + + if "statusCode" in docs_conversion_lambda_response_json and \ + docs_conversion_lambda_response_json["statusCode"] == 200: + bucket_name = docs_conversion_lambda_response_json["bucket"] + file_path = docs_conversion_lambda_response_json["file"] + filename = file_path.split("/")[-1] + + if download_file(file_path, bucket_name, f"/tmp/{filename}"): + s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( + f"/tmp/{filename}", file_name + ) + if s3_file_path: + extraction_status = ExtractionStatus.SUCCESS.value + else: + extraction_status = ExtractionStatus.FAILED.value + else: + flag = True + else: + logging.error(f"Error occurred during file conversion. {docs_conversion_lambda_response_json['error']}") + flag = True + else: + logging.warn("Could not upload the file to s3.") + flag = True + + if flag: + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + elif content_type == UrlTypes.IMG.value: + logging.warn("Text extraction from Images is not available.") + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + else: + logging.error("Text extraction is not available for this datatype.") + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value + + logging.info(f"The extracted file path is {s3_file_path}") + logging.info(f"The extracted image path is {s3_images_path}") + logging.info(f"The status of the extraction is {str(extraction_status)}") + + return s3_file_path, s3_images_path, str(total_pages), str(total_words_count), str(extraction_status) + + +def process_docs(): + logging.info(f"Processing the {URL}") + s3_text_path, s3_images_path, total_pages, total_words_count, extraction_status = handle_urls(URL) + + sqs_message = { + "client_id": CLIENT_ID, + "url": URL, + "callback_url": CALLBACK_URL, + "s3_text_path": s3_text_path, + "s3_images_path": s3_images_path, + "total_pages": total_pages, + "total_words_count": total_words_count, + "extraction_status": extraction_status + } + logging.info(sqs_message) + send_message2sqs(**sqs_message) + + +process_docs() diff --git a/lambda_fns/extract_docs/poetry.lock b/lambda_fns/extract_docs/poetry.lock new file mode 100644 index 00000000..cd19a4aa --- /dev/null +++ b/lambda_fns/extract_docs/poetry.lock @@ -0,0 +1,705 @@ +[[package]] +name = "anytree" +version = "2.8.0" +description = "Powerful and Lightweight Python Tree Data Structure.." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +dev = ["check-manifest"] +test = ["coverage"] + +[[package]] +name = "beautifulsoup4" +version = "4.11.1" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "boto3" +version = "1.19.0" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +botocore = ">=1.22.0,<1.23.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.5.0,<0.6.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.22.12" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +jmespath = ">=0.7.1,<1.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.12.5)"] + +[[package]] +name = "bs4" +version = "0.0.1" +description = "Dummy package for Beautiful Soup" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +beautifulsoup4 = "*" + +[[package]] +name = "certifi" +version = "2021.5.30" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.4" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "cssselect" +version = "1.1.0" +description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "deep-parser" +version = "0.1.0" +description = "This package contains parsing utilities for text extraction" +category = "main" +optional = false +python-versions = "*" +develop = false + +[package.dependencies] +anytree = "*" +beautifulsoup4 = "*" +func-timeout = "*" +lxml = "*" +numpy = "*" +PyMuPDF = "*" +pyppeteer = "0.0.14" +pyyaml = "*" +readable-content = "*" +requests-html = "*" + +[package.source] +type = "git" +url = "https://github.com/the-deep/deepex.git" +reference = "master" +resolved_reference = "cee0daf673f1e1ead2a734880130a2e1b09e7fd6" + +[[package]] +name = "fake-useragent" +version = "0.1.11" +description = "Up to date simple useragent faker with real world database" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "func-timeout" +version = "4.3.5" +description = "Python module which allows you to specify timeouts when calling any existing function. Also provides support for stoppable-threads" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "idna" +version = "3.2" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "jmespath" +version = "0.10.0" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "lxml" +version = "4.9.1" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["beautifulsoup4"] +source = ["Cython (>=0.29.7)"] + +[[package]] +name = "numpy" +version = "1.21.2" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7,<3.11" + +[[package]] +name = "parse" +version = "1.19.0" +description = "parse() is the opposite of format()" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "pillow" +version = "8.3.2" +description = "Python Imaging Library (Fork)" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pyee" +version = "9.0.4" +description = "A port of node.js's EventEmitter to python." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +typing-extensions = "*" + +[[package]] +name = "pymupdf" +version = "1.18.17" +description = "Python bindings for the PDF toolkit and renderer MuPDF" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "pyppeteer" +version = "0.0.14" +description = "Headless chrome/chromium automation library (unofficial port of puppeteer)" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +pyee = "*" +websockets = "*" + +[[package]] +name = "pyquery" +version = "1.4.3" +description = "A jquery-like library for python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cssselect = ">0.7.9" +lxml = ">=2.1" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[[package]] +name = "readable-content" +version = "0.1.2" +description = "Collect actual content of any article, blog, news, etc." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +beautifulsoup4 = "*" +lxml = "*" + +[[package]] +name = "requests" +version = "2.26.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + +[[package]] +name = "requests-html" +version = "0.10.0" +description = "HTML Parsing for Humans." +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +bs4 = "*" +fake-useragent = "*" +parse = "*" +pyppeteer = ">=0.0.14" +pyquery = "*" +requests = "*" +w3lib = "*" + +[[package]] +name = "s3transfer" +version = "0.5.2" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "sentry-sdk" +version = "1.5.8" +description = "Python client for Sentry (https://sentry.io)" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +flask = ["flask (>=0.11)", "blinker (>=1.1)"] +httpx = ["httpx (>=0.16.0)"] +pure_eval = ["pure-eval", "executing", "asttokens"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["quart (>=0.16.1)", "blinker (>=1.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +tornado = ["tornado (>=5)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "soupsieve" +version = "2.3.2.post1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.6" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "w3lib" +version = "1.22.0" +description = "Library of web-related functions" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.4.1" + +[[package]] +name = "websockets" +version = "10.3" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" +optional = false +python-versions = ">=3.7" + +[metadata] +lock-version = "1.1" +python-versions = ">=3.10, <3.11" +content-hash = "6f9776c669bb795a23828b4f2c7cb765a4901712de650f6ceef0f10fd1778f65" + +[metadata.files] +anytree = [ + {file = "anytree-2.8.0-py2.py3-none-any.whl", hash = "sha256:14c55ac77492b11532395049a03b773d14c7e30b22aa012e337b1e983de31521"}, + {file = "anytree-2.8.0.tar.gz", hash = "sha256:3f0f93f355a91bc3e6245319bf4c1d50e3416cc7a35cc1133c1ff38306bbccab"}, +] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, + {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, +] +boto3 = [ + {file = "boto3-1.19.0-py3-none-any.whl", hash = "sha256:d468b1f63f22ccd6b4bfbdebe6fd0c0b4620f38276af965ed139fe3eb85d16bb"}, + {file = "boto3-1.19.0.tar.gz", hash = "sha256:f93fed6153f7def66f1b17e6794c6ec3bec46229b213d3fa63f1eca126f5e992"}, +] +botocore = [ + {file = "botocore-1.22.12-py3-none-any.whl", hash = "sha256:1d1094fb53ebe4535d8840fbd7c14aadb65bde7ff03a65f9a4f1d76bd03e16ff"}, + {file = "botocore-1.22.12.tar.gz", hash = "sha256:fc59b55e8c5dde64b017b2f114c25f8cce397b667e812aea7eafb4b59b49d7cb"}, +] +bs4 = [ + {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"}, +] +certifi = [ + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, + {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, +] +cssselect = [ + {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, + {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"}, +] +deep-parser = [] +fake-useragent = [ + {file = "fake-useragent-0.1.11.tar.gz", hash = "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"}, +] +func-timeout = [ + {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"}, +] +idna = [ + {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, + {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, +] +jmespath = [ + {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, + {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, +] +lxml = [] +numpy = [ + {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13"}, + {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218"}, + {file = "numpy-1.21.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1"}, + {file = "numpy-1.21.2-cp37-cp37m-win32.whl", hash = "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310"}, + {file = "numpy-1.21.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f"}, + {file = "numpy-1.21.2-cp38-cp38-win32.whl", hash = "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557"}, + {file = "numpy-1.21.2-cp38-cp38-win_amd64.whl", hash = "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f"}, + {file = "numpy-1.21.2-cp39-cp39-win32.whl", hash = "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad"}, + {file = "numpy-1.21.2-cp39-cp39-win_amd64.whl", hash = "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724"}, + {file = "numpy-1.21.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7"}, + {file = "numpy-1.21.2.zip", hash = "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc"}, +] +parse = [ + {file = "parse-1.19.0.tar.gz", hash = "sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b"}, +] +pillow = [ + {file = "Pillow-8.3.2-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4"}, + {file = "Pillow-8.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2"}, + {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f"}, + {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c"}, + {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a"}, + {file = "Pillow-8.3.2-cp310-cp310-win32.whl", hash = "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228"}, + {file = "Pillow-8.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875"}, + {file = "Pillow-8.3.2-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2"}, + {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8"}, + {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b"}, + {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629"}, + {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7"}, + {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550"}, + {file = "Pillow-8.3.2-cp36-cp36m-win32.whl", hash = "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073"}, + {file = "Pillow-8.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196"}, + {file = "Pillow-8.3.2-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71"}, + {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83"}, + {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba"}, + {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1"}, + {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b"}, + {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da"}, + {file = "Pillow-8.3.2-cp37-cp37m-win32.whl", hash = "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9"}, + {file = "Pillow-8.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3"}, + {file = "Pillow-8.3.2-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616"}, + {file = "Pillow-8.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3"}, + {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979"}, + {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4"}, + {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30"}, + {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6"}, + {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b"}, + {file = "Pillow-8.3.2-cp38-cp38-win32.whl", hash = "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341"}, + {file = "Pillow-8.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb"}, + {file = "Pillow-8.3.2-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9"}, + {file = "Pillow-8.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630"}, + {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056"}, + {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6"}, + {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d"}, + {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b"}, + {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441"}, + {file = "Pillow-8.3.2-cp39-cp39-win32.whl", hash = "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09"}, + {file = "Pillow-8.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19"}, + {file = "Pillow-8.3.2-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864"}, + {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa"}, + {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd"}, + {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624"}, + {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1"}, + {file = "Pillow-8.3.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96"}, + {file = "Pillow-8.3.2.tar.gz", hash = "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c"}, +] +pyee = [ + {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"}, + {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, +] +pymupdf = [ + {file = "PyMuPDF-1.18.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4546e426e00e3ace148adaf95e0952061391ea04ec285c7a4592e11b9a372d0b"}, + {file = "PyMuPDF-1.18.17-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0dac1bc4c474dce4d0569b02eb11126f15d9a12d4855fddf21471cff75db493e"}, + {file = "PyMuPDF-1.18.17-cp310-cp310-win32.whl", hash = "sha256:8e7d1036a135ecbe30d185d968556367b018c9954043e17408f1798c1d0a7685"}, + {file = "PyMuPDF-1.18.17-cp310-cp310-win_amd64.whl", hash = "sha256:87f5af9570e6b12191b05e23dfd3e1fd8940f2c0a1ec0773c6496205fdf24f7c"}, + {file = "PyMuPDF-1.18.17-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3c7e01284d9832c3ba76f6eb5c6825827f26749eb20cfe20f633bf980b3d3d85"}, + {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ed93f0964b49033073704e4bfb0908046c799731ee1e357f2b4ec934e551b515"}, + {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c76115e81a8a1596fe127b9b8b45ec846cb6bcfa891460a2efad4eb145573a4"}, + {file = "PyMuPDF-1.18.17-cp36-cp36m-win32.whl", hash = "sha256:615ed1e4d13d404e219c4caff0c8c7e5557a331ffaeb162e12016562ff160178"}, + {file = "PyMuPDF-1.18.17-cp36-cp36m-win_amd64.whl", hash = "sha256:3e805320eb1c6d4957d66b06baa4c2726fd0bb84acd5114be08a69c0a071abe6"}, + {file = "PyMuPDF-1.18.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d0b1a211bea5f98556dec57172675958e28a90818d44dc6b095efb37034854b8"}, + {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57a8c34a236e7021f6defda501eb2afa310687fd45cfb650211d09e9ed4252e7"}, + {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecacfef4760c8b451284940d5dbd9aeeb36fd0c1e495545c4f487196047e167d"}, + {file = "PyMuPDF-1.18.17-cp37-cp37m-win32.whl", hash = "sha256:20591b790035f5d0cebfcb0d12f336e7d2bab1d708d20a20c8a75e8c6bb51481"}, + {file = "PyMuPDF-1.18.17-cp37-cp37m-win_amd64.whl", hash = "sha256:a89a528e3218026ffb8ccceffaa813b4a0bb2e4b8edb3a44371b0f697b18349c"}, + {file = "PyMuPDF-1.18.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9212f95bb2978ddf3669dd8feef09ea47c13603b58efee6a305b6fe4a8a6a621"}, + {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3c605f2f2936067f9e2ce7d76127bb040c4528a347222ae23d7f440af8965afd"}, + {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a21735d8306d058adb62dcdd71d70f32c614f23729abc3971c1eb9099712069c"}, + {file = "PyMuPDF-1.18.17-cp38-cp38-win32.whl", hash = "sha256:c385f38bc7d299917c3c64a096447d8fb573373705c120efec316fd109f53de8"}, + {file = "PyMuPDF-1.18.17-cp38-cp38-win_amd64.whl", hash = "sha256:9fca5a5666e3a09c9b573d2edd230a0e51c379d1b9e11f7121c622f5795fcce5"}, + {file = "PyMuPDF-1.18.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e4581e2bbee1392cac203022e93de259221f9dd2ebfd1ca7e2ac1f00d454cff"}, + {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ee986ff5ed40b633b7e4bf321658ab31f1285b01628e2830c4bddf0003677f36"}, + {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ac77aac8db26dfb118dcc1b5f7857f7b669e9a415a76cc2fb77fd8429af2d4f"}, + {file = "PyMuPDF-1.18.17-cp39-cp39-win32.whl", hash = "sha256:4379bf78ecf4ba17291ba01582ca02e5ddbbc96004b31e8be1cc1b582795b880"}, + {file = "PyMuPDF-1.18.17-cp39-cp39-win_amd64.whl", hash = "sha256:b7dbdfa078069413b3ef442f2b21770477bb7c1ce724b4b332198ca7bc3efffe"}, + {file = "PyMuPDF-1.18.17.tar.gz", hash = "sha256:fa39ee5e91eae77818e07b6bb7e0cb0b402ad88e39a74b08626ce1c2150c5414"}, +] +pyppeteer = [ + {file = "pyppeteer-0.0.14.tar.gz", hash = "sha256:4e0409fb30bb717296432d5548f6a3407d78d9efcf7a17f308fdb42d43607d9f"}, +] +pyquery = [ + {file = "pyquery-1.4.3-py3-none-any.whl", hash = "sha256:1fc33b7699455ed25c75282bc8f80ace1ac078b0dda5a933dacbd8b1c1f83963"}, + {file = "pyquery-1.4.3.tar.gz", hash = "sha256:a388eefb6bc4a55350de0316fbd97cda999ae669b6743ae5b99102ba54f5aa72"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pyyaml = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] +readable-content = [ + {file = "readable-content-0.1.2.tar.gz", hash = "sha256:94e6c41c95db814473a190170c861d7a2a20ce04b245c2ecbd20da9eef1f8f93"}, +] +requests = [ + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, +] +requests-html = [ + {file = "requests-html-0.10.0.tar.gz", hash = "sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947"}, + {file = "requests_html-0.10.0-py3-none-any.whl", hash = "sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2"}, +] +s3transfer = [ + {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"}, + {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"}, +] +sentry-sdk = [] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +soupsieve = [ + {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, + {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, +] +typing-extensions = [] +urllib3 = [ + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, +] +w3lib = [ + {file = "w3lib-1.22.0-py2.py3-none-any.whl", hash = "sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53"}, + {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, +] +websockets = [ + {file = "websockets-10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978"}, + {file = "websockets-10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500"}, + {file = "websockets-10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8"}, + {file = "websockets-10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f"}, + {file = "websockets-10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47"}, + {file = "websockets-10.3-cp310-cp310-win32.whl", hash = "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae"}, + {file = "websockets-10.3-cp310-cp310-win_amd64.whl", hash = "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079"}, + {file = "websockets-10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79"}, + {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e"}, + {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6"}, + {file = "websockets-10.3-cp37-cp37m-win32.whl", hash = "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1"}, + {file = "websockets-10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36"}, + {file = "websockets-10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69"}, + {file = "websockets-10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c"}, + {file = "websockets-10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442"}, + {file = "websockets-10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76"}, + {file = "websockets-10.3-cp38-cp38-win32.whl", hash = "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559"}, + {file = "websockets-10.3-cp38-cp38-win_amd64.whl", hash = "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094"}, + {file = "websockets-10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667"}, + {file = "websockets-10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680"}, + {file = "websockets-10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3"}, + {file = "websockets-10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8"}, + {file = "websockets-10.3-cp39-cp39-win32.whl", hash = "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582"}, + {file = "websockets-10.3-cp39-cp39-win_amd64.whl", hash = "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02"}, + {file = "websockets-10.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4"}, + {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755"}, + {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, + {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, +] diff --git a/lambda_fns/extract_docs/pyproject.toml b/lambda_fns/extract_docs/pyproject.toml new file mode 100644 index 00000000..34a23fc0 --- /dev/null +++ b/lambda_fns/extract_docs/pyproject.toml @@ -0,0 +1,27 @@ +[tool.poetry] +name = "deepl-parser" +version = "0.1.0" +description = "" +authors = ["deep-dev "] + +[tool.poetry.dependencies] +python = ">=3.10, <3.11" +anytree = "==2.8.0" +certifi = "==2021.5.30" +charset-normalizer = "==2.0.4" +idna = "==3.2" +numpy = "==1.21.2" +PyMuPDF = "==1.18.17" +PyYAML = "==5.4.1" +requests = "==2.26.0" +urllib3 = "==1.26.6" +Pillow = "==8.3.2" +boto3 = "==1.19.0" +deep-parser = { git = "https://github.com/the-deep/deepex.git", branch = "master" } +sentry-sdk = "==1.5.8" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 2902c1ff74418e25780a09fc2cb8d56d4a1a37a4 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 4 Aug 2022 11:20:04 +0545 Subject: [PATCH 17/45] reduce the number of privisioned lambda; --- modules/reserved_sqs_lambda_extract_docs/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index 1529359a..ec539935 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -145,7 +145,7 @@ module "reserved_extract_docs_fn" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 10 + provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 reserved_concurrent_executions = 30 build_in_docker = true @@ -177,8 +177,8 @@ module "reserved_extract_docs_fn" { # } resource "aws_appautoscaling_target" "reserved_extract_docs_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 10 - min_capacity = var.environment == "dev" ? 1 : 5 + max_capacity = var.environment == "dev" ? 1 : 5 + min_capacity = var.environment == "dev" ? 1 : 3 resource_id = "function:${module.reserved_extract_docs_fn.lambda_function_name}:${module.reserved_extract_docs_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" From 83994548211f6d32f0b4f72bd7d911cd34e279e5 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 5 Aug 2022 14:25:47 +0545 Subject: [PATCH 18/45] try..except block used; --- lambda_fns/extract_docs/content_types.py | 6 +++++- lambda_fns/extract_docs/deepex_ecs/app.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index bacc8682..17411bb6 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -64,7 +64,11 @@ def get_content_type(self, url, req_headers): url.endswith(".gif") or url.endswith(".bmp") or content_type in self.content_types_img: return UrlTypes.IMG.value else: - temp_filepath = download(url, out="/tmp/") + try: + temp_filepath = download(url, out="/tmp/") + except Exception as e: + logging.error(f"Error while downloading the file from {url} to check the file extension.") + return None if os.path.exists(temp_filepath): os.remove(temp_filepath) if temp_filepath.endswith(".pdf"): diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index 708ac659..67c522be 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -374,7 +374,7 @@ def handle_urls(url): s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value else: - logging.error("Text extraction is not available for this datatype.") + logging.error(f"Text extraction is not available for this content type - {content_type}") s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value From 1914f136e2cca4f423faf741050c02a76ca0f03c Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 5 Aug 2022 15:02:37 +0545 Subject: [PATCH 19/45] param values added for prod env --- prod.tfvars | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/prod.tfvars b/prod.tfvars index 11accef3..df7ae5ea 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -23,4 +23,17 @@ docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" docs_convert_bucket_name = "deep-large-docs-conversion" # sentry url -sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file +sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" + +# VPC +az_count = 1 +cidr_block = "172.18.0.0/16" + +# ECS role +ecs_task_execution_role = "ECSTaskExecutionRole" + +# ECS +fargate_cpu = "2048" +fargate_memory = "4096" +app_count = 1 +app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser" \ No newline at end of file From 4acc48a68d02b2edd9724ac7e3347c8ddc43da6e Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 1 Sep 2022 14:14:25 +0545 Subject: [PATCH 20/45] config values for dev env; --- dev.tfvars | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/dev.tfvars b/dev.tfvars index 5d671977..23fce45d 100644 --- a/dev.tfvars +++ b/dev.tfvars @@ -23,4 +23,17 @@ docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" docs_convert_bucket_name = "deep-large-docs-conversion" # sentry url -sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" \ No newline at end of file +sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" + +# VPC +az_count = 1 +cidr_block = "172.16.0.0/16" + +# ECS role +ecs_task_execution_role = "ECSTaskExecutionRole" + +# ECS +fargate_cpu = "2048" +fargate_memory = "4096" +app_count = 1 +app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser" From 708b73a9e005363210f4a7750714e1a1f819b3da Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 1 Sep 2022 16:34:17 +0545 Subject: [PATCH 21/45] code refactored in parser for text extraction; --- lambda_fns/extract_docs/Dockerfile | 2 +- lambda_fns/extract_docs/app.py | 4 +- lambda_fns/extract_docs/content_types.py | 53 ++--- lambda_fns/extract_docs/deepex_ecs/app.py | 224 ++++++++++------------ lambda_fns/extract_docs/s3_handler.py | 188 ++++++++++++++++++ 5 files changed, 322 insertions(+), 149 deletions(-) create mode 100644 lambda_fns/extract_docs/s3_handler.py diff --git a/lambda_fns/extract_docs/Dockerfile b/lambda_fns/extract_docs/Dockerfile index f661b7ab..b13e8b2e 100644 --- a/lambda_fns/extract_docs/Dockerfile +++ b/lambda_fns/extract_docs/Dockerfile @@ -25,4 +25,4 @@ RUN apt-get update -y \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* -COPY deepex_ecs/app.py content_types.py wget.py /code/ +COPY deepex_ecs/app.py content_types.py wget.py s3_handler.py /code/ diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 7681545d..896c9898 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -78,8 +78,8 @@ def store_text_s3(data, bucket_name, key): def upload_file_to_s3( url, - key="temporaryfile.pdf", - bucketname="deep-large-docs-conversion" + key, + bucketname ): try: session = boto3.Session() diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index 17411bb6..ad3cd739 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -60,8 +60,13 @@ def get_content_type(self, url, req_headers): return UrlTypes.PPTX.value elif url.endswith(".ppt") or content_type in self.content_types_ppt: return UrlTypes.PPT.value - elif url.endswith(".jpg") or url.endswith(".jpeg") or url.endswith(".png") or \ - url.endswith(".gif") or url.endswith(".bmp") or content_type in self.content_types_img: + elif (content_type in self.content_types_img or + any([ + url.endswith(f".{extension}") for extension in [ + "jpg", "jpeg", "png", "gif", "bmp" + ] + ]) + ): return UrlTypes.IMG.value else: try: @@ -69,28 +74,26 @@ def get_content_type(self, url, req_headers): except Exception as e: logging.error(f"Error while downloading the file from {url} to check the file extension.") return None - if os.path.exists(temp_filepath): - os.remove(temp_filepath) - if temp_filepath.endswith(".pdf"): - return UrlTypes.PDF.value - elif temp_filepath.endswith(".docx"): - return UrlTypes.DOCX.value - elif temp_filepath.endswith(".doc"): - return UrlTypes.MSWORD.value - elif temp_filepath.endswith(".xlsx"): - return UrlTypes.XLSX.value - elif temp_filepath.endswith(".xls"): - return UrlTypes.XLS.value - elif temp_filepath.endswith(".pptx"): - return UrlTypes.PPTX.value - elif temp_filepath.endswith(".ppt"): - return UrlTypes.PPT.value - elif temp_filepath.endswith(".jpg") or temp_filepath.endswith(".jpeg") or temp_filepath.endswith(".png") or \ - temp_filepath.endswith(".gif") or temp_filepath.endswith(".bmp"): - return UrlTypes.IMG.value - else: - logging.warn(f'Could not determine the content-type of the {url}') + EXTENSION_TO_ENUM_MAP = { + "pdf": UrlTypes.PDF, + "docx": UrlTypes.DOCX, + "doc": UrlTypes.MSWORD, + "xlsx": UrlTypes.XLSX, + "xls": UrlTypes.XLS, + "pptx": UrlTypes.PPTX, + "ppt": UrlTypes.PPT, + # Images + "jpg": UrlTypes.IMG, + "jpeg": UrlTypes.IMG, + "png": UrlTypes.IMG, + "gif": UrlTypes.IMG, + "bmp": UrlTypes.IMG, + } + file_extension = temp_filepath.split(".")[-1] + if file_extension not in EXTENSION_TO_ENUM_MAP: + logging.warn(f"Could not determine the content-type of the {url}") return None - except requests.exceptions.RequestException: - logging.error(f'Exception occurred. Could not determine the content-type of the {url}') + return EXTENSION_TO_ENUM_MAP[file_extension].value + except requests.exceptions.RequestException as e: + logging.error(f"Exception occurred: {str(e)}. Could not determine the content-type of the {url}.", exc_info=True) return None diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index 67c522be..bd135d9a 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -19,8 +19,10 @@ try: from content_types import ExtractContentType, UrlTypes + from s3_handler import Storage except ImportError: from .content_types import ExtractContentType, UrlTypes + from .s3_handler import Storage logging.getLogger().setLevel(logging.INFO) @@ -58,6 +60,37 @@ class ExtractionStatus(Enum): SUCCESS = 1 +def invoke_conversion_lambda(tmp_filename, ext_type): + payload = json.dumps({ + "file": tmp_filename, + "bucket": DOCS_CONVERSION_BUCKET_NAME, + "ext": ext_type, + "fromS3": 1 + }) + + docs_conversion_lambda_response = lambda_client.invoke( + FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, + InvocationType="RequestResponse", + Payload=payload + ) + docs_conversion_lambda_response_json = json.loads( + docs_conversion_lambda_response["Payload"].read().decode("utf-8") + ) + return docs_conversion_lambda_response_json + + +def create_tempfile(response): + tempf = tempfile.NamedTemporaryFile(mode="w+b") + for chunk in response.iter_content(chunk_size=128): + tempf.write(chunk) + tempf.seek(0) + return tempf + +def preprocess_extracted_texts(text): + extracted_text = text.replace("\x00", "") # remove null chars + extracted_text = extracted_text.encode('utf-8', 'ignore').decode('utf-8') + return extracted_text + def extract_path(filepath): """ Extracts bucket and key names from the s3 URI @@ -71,18 +104,10 @@ def extract_path(filepath): return None, None, None -def store_text_s3(data, bucket_name, key): - s3_client.put_object( - Body=data, - Bucket=bucket_name, - Key=key - ) - - def upload_file_to_s3( url, - key="temporaryfile.pdf", - bucketname="deep-large-docs-conversion" + key, + bucketname ): try: session = boto3.Session() @@ -178,7 +203,13 @@ def send_message2sqs( logging.error("Message not sent to the processed SQS.") -def get_extracted_content_links(file_path, file_name): +def generate_s3_path(s3_path_prefix, file_name): + s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" + s3_images_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/images" + return s3_file_path, s3_images_path + + +def get_extracted_text(file_path, file_name): try: with open(pathlib.Path(file_path), "rb") as f: binary = base64.b64encode(f.read()) @@ -186,14 +217,14 @@ def get_extracted_content_links(file_path, file_name): document = TextFromFile(stream=binary, ext="pdf") entries, images = document.serial_extract_text(output_format="list") except Exception as e: - logging.error(f"Extraction failed: {str(e)}") - return None, None, -1, -1 + logging.error(f"Extraction failed: {str(e)}", exc_info=True) + return None, None, -1, -1, 0 entries_list = [item for sublist in entries for item in sublist] extracted_text = "\n".join(entries_list) - extracted_text = extracted_text.replace("\x00", "") # remove null chars - extracted_text = extracted_text.encode('utf-8', 'ignore').decode('utf-8') + extracted_text = preprocess_extracted_texts(extracted_text) + total_pages = len(entries) total_words_count = get_words_count(extracted_text) @@ -203,14 +234,17 @@ def get_extracted_content_links(file_path, file_name): s3_path_prefix = pathlib.Path(date_today, dir_name) - store_text_s3( - extracted_text, - DEST_BUCKET_NAME, - f"{str(s3_path_prefix)}/{file_name}" - ) + # Upload file to the s3 bucket + s3_uploader = Storage(DEST_BUCKET_NAME, "") + tempf = tempfile.NamedTemporaryFile(mode="w") + tempf.write(extracted_text) + tempf.seek(0) - local_temp_directory = pathlib.Path('/tmp', file_name) - local_temp_directory.mkdir(parents=True) if not local_temp_directory.exists() else None + with open(tempf.name, "rb") as f: + file_path = s3_uploader.upload(f"{str(s3_path_prefix)}/{file_name}", f) + + #local_temp_directory = pathlib.Path('/tmp', file_name) + #local_temp_directory.mkdir(parents=True) if not local_temp_directory.exists() else None # Note: commented for now # images.save_images(directory_path=local_temp_directory) @@ -224,24 +258,22 @@ def get_extracted_content_links(file_path, file_name): # f"{str(s3_path_prefix)}/images/{f}" # ) - s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" - s3_images_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/images" + s3_file_path, s3_images_path = generate_s3_path(s3_path_prefix, file_name) + return s3_file_path, s3_images_path, total_pages, total_words_count, 1 - return s3_file_path, s3_images_path, total_pages, total_words_count - -def get_extracted_text_web_links(link, file_name): +def get_extracted_text_html_links(link, file_name): try: web_text = TextFromWeb(url=link) entries = web_text.extract_text(output_format="list") except Exception as e: - logging.error(f"Extraction from website failed {e}") - return None, None, -1, -1 + logging.error(f"Extraction from website failed {e}", exc_info=True) + return None, None, -1, -1, 0 entries_list = [item for sublist in entries for item in sublist] extracted_text = "\n".join(entries_list) - extracted_text = extracted_text.replace("\x00", "") # remove null chars + extracted_text = preprocess_extracted_texts(extracted_text) total_pages = 1 total_words_count = get_words_count(extracted_text) @@ -252,13 +284,17 @@ def get_extracted_text_web_links(link, file_name): s3_path_prefix = pathlib.Path(date_today, dir_name) - store_text_s3( - extracted_text, - DEST_BUCKET_NAME, - f"{str(s3_path_prefix)}/{file_name}" - ) - s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" - return s3_file_path, None, total_pages, total_words_count # No images extraction (lib doesn't support?) + # Upload file to the s3 bucket + s3_uploader = Storage(DEST_BUCKET_NAME, "") + tempf = tempfile.NamedTemporaryFile(mode="w") + tempf.write(extracted_text) + tempf.seek(0) + + with open(tempf.name, "rb") as f: + file_path = s3_uploader.upload(f"{str(s3_path_prefix)}/{file_name}", f) + + s3_file_path, _ = generate_s3_path(s3_path_prefix, file_name) # image path is not required as is not supported + return s3_file_path, None, total_pages, total_words_count, 1 # No images extraction (lib doesn't support?) def handle_urls(url): @@ -266,111 +302,57 @@ def handle_urls(url): file_name = EXTRACTED_FILE_NAME - if url.startswith("s3"): - bucket_name, file_path, file_name = extract_path(url) - local_file_path = f"/tmp/{file_name}" - s3_client.download_file( - bucket_name, - file_path, - local_file_path - ) - try: - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( - local_file_path, file_name - ) - extraction_status = ExtractionStatus.SUCCESS.value - except Exception as e: - logging.error(e, exc_info=True) - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value - - elif content_type == UrlTypes.PDF.value: # assume it is http/https pdf weblink - s3_file_path = None - s3_images_path = None - + if content_type == UrlTypes.PDF.value: # assume it is http/https pdf weblink response = requests.get(url, headers=REQ_HEADERS, stream=True) - with tempfile.NamedTemporaryFile(mode='w+b') as temp: - for chunk in response.iter_content(chunk_size=128): - temp.write(chunk) - temp.seek(0) - - try: - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( - temp.name, file_name - ) - if s3_file_path: - extraction_status = ExtractionStatus.SUCCESS.value - else: - extraction_status = ExtractionStatus.FAILED.value - except Exception as e: - logging.error(f"Error occurred during text extraction. {str(e)}", exc_info=True) - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value - elif content_type == UrlTypes.HTML.value: # assume it is a static webpage - try: - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_text_web_links( - url, file_name - ) - if s3_file_path: - extraction_status = ExtractionStatus.SUCCESS.value - else: - extraction_status = ExtractionStatus.FAILED.value - except Exception: - logging.error(f"Error occurred during text extraction. {str(e)}", exc_info=True) - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value - elif content_type == UrlTypes.DOCX.value or content_type == UrlTypes.MSWORD.value or \ - content_type == UrlTypes.XLSX.value or content_type == UrlTypes.XLS.value or \ - content_type == UrlTypes.PPTX.value or content_type == UrlTypes.PPT.value: + tempf = create_tempfile(response) + s3_file_path, s3_images_path, total_pages, total_words_count, extraction_status = get_extracted_text( + tempf.name, file_name + ) + elif content_type == UrlTypes.HTML.value: # assume it is a static webpage + s3_file_path, s3_images_path, total_pages, total_words_count, extraction_status = get_extracted_text_html_links( + url, file_name + ) + elif content_type in [ + UrlTypes.DOCX.value, + UrlTypes.MSWORD.value, + UrlTypes.XLSX.value, + UrlTypes.XLS.value, + UrlTypes.PPTX.value, + UrlTypes.PPT.value + ]: ext_type = content_type tmp_filename = f"{uuid.uuid4().hex}.{ext_type}" flag = False - if upload_file_to_s3(url, key=tmp_filename, bucketname=DOCS_CONVERSION_BUCKET_NAME): - payload = json.dumps({ - "file": tmp_filename, - "bucket": DOCS_CONVERSION_BUCKET_NAME, - "ext": ext_type, - "fromS3": 1 - }) - - docs_conversion_lambda_response = lambda_client.invoke( - FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, - InvocationType="RequestResponse", - Payload=payload - ) - docs_conversion_lambda_response_json = json.loads( - docs_conversion_lambda_response["Payload"].read().decode("utf-8") - ) - - if "statusCode" in docs_conversion_lambda_response_json and \ - docs_conversion_lambda_response_json["statusCode"] == 200: + response = requests.get(url, headers=REQ_HEADERS, stream=True) + + s3_uploader = Storage(DOCS_CONVERSION_BUCKET_NAME, "") + tempf = create_tempfile(response) + with open(tempf.name, "rb") as f: + s3_uploader.upload(tmp_filename, f) + # Converts docx, xlsx, doc, xls, ppt, pptx type files to pdf using lambda + docs_conversion_lambda_response_json = invoke_conversion_lambda(tmp_filename, ext_type) + + if ("statusCode" in docs_conversion_lambda_response_json and + docs_conversion_lambda_response_json["statusCode"] == 200): bucket_name = docs_conversion_lambda_response_json["bucket"] file_path = docs_conversion_lambda_response_json["file"] filename = file_path.split("/")[-1] if download_file(file_path, bucket_name, f"/tmp/{filename}"): - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( + s3_file_path, s3_images_path, total_pages, total_words_count, extraction_status = get_extracted_text( f"/tmp/{filename}", file_name ) - if s3_file_path: - extraction_status = ExtractionStatus.SUCCESS.value - else: - extraction_status = ExtractionStatus.FAILED.value else: flag = True else: logging.error(f"Error occurred during file conversion. {docs_conversion_lambda_response_json['error']}") flag = True - else: - logging.warn("Could not upload the file to s3.") - flag = True - if flag: s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value elif content_type == UrlTypes.IMG.value: - logging.warn("Text extraction from Images is not available.") + logging.warning("Text extraction from Images is not available.") s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 extraction_status = ExtractionStatus.FAILED.value else: diff --git a/lambda_fns/extract_docs/s3_handler.py b/lambda_fns/extract_docs/s3_handler.py new file mode 100644 index 00000000..16212188 --- /dev/null +++ b/lambda_fns/extract_docs/s3_handler.py @@ -0,0 +1,188 @@ +# SOURCE: https://github.com/django/django/blob/master/django/core/files/storage.py +# SOURCE: https://github.com/jschneier/django-storages/blob/master/storages/backends/s3boto3.py +# NOTE: Only copied used part. + +import io +import os +import secrets +import boto3 +import posixpath +import mimetypes +from gzip import GzipFile +from tempfile import SpooledTemporaryFile +from botocore.exceptions import ClientError + + +def get_random_string(length, allowed_chars=( + 'abcdefghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +)): + """ + SOURCE: DJANGO + Return a securely generated random string. + The bit length of the returned value can be calculated with the formula: + log_2(len(allowed_chars)^length) + For example, with default `allowed_chars` (26+26+10), this gives: + * length: 12, bit length =~ 71 bits + * length: 22, bit length =~ 131 bits + """ + return ''.join(secrets.choice(allowed_chars) for i in range(length)) + +def force_bytes(s): + if isinstance(s, bytes): + return s + if isinstance(s, memoryview): + return bytes(s) + return str(s).encode('utf-8', 'strict') + + +def safe_join(base, *paths): + base_path = base + base_path = base_path.rstrip('/') + paths = [p for p in paths] + final_path = base_path + '/' + for path in paths: + _final_path = posixpath.normpath(posixpath.join(final_path, path)) + # posixpath.normpath() strips the trailing /. Add it back. + if path.endswith('/') or _final_path + '/' == final_path: + _final_path += '/' + final_path = _final_path + if final_path == base_path: + final_path += '/' + base_path_len = len(base_path) + if (not final_path.startswith(base_path) or final_path[base_path_len] != '/'): + raise ValueError('the joined path is located outside of the base path component') + return final_path.lstrip('/') + + +class Storage(): + DEFAULT_CONTENT_TYPE = 'application/octet-stream' + DEFAULT_QUERYSTRING_EXPIRE = 3600 # 1 hour + + QUERYSTRING_AUTH = True + GZIP_CONTENT_TYPES = [ + 'text/javascript', 'application/javascript', 'application/x-javascript', + 'image/svg+xml', 'application/json', 'text/css', + ] + + def __init__(self, bucket_name, base_path): + self.bucket_name = bucket_name + self.base_path = base_path + self._file = None + + @property + def connection(self): + connection = getattr(self, '_connection', None) + if connection is None: + session = boto3.session.Session() + self._connection = session.resource('s3') + return self._connection + + @property + def bucket(self): + bucket = getattr(self, '_bucket', None) + if bucket is None: + self._bucket = self.connection.Bucket(self.bucket_name) + return self._bucket + + def exists(self, name): + try: + self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name) + return True + except ClientError: + return False + + def get_alternative_name(self, file_root, file_ext): + return '%s_%s%s' % (file_root, get_random_string(7), file_ext) + + def get_available_name(self, name, max_length=1024): + dir_name, file_name = os.path.split(name) + file_root, file_ext = os.path.splitext(file_name) + while self.exists(name) or (max_length and len(name) > max_length): + name = os.path.join(dir_name, self.get_alternative_name(file_root, file_ext)) + if max_length is None: + continue + truncation = len(name) - max_length + if truncation > 0: + file_root = file_root[:-truncation] + if not file_root: + raise Exception(f"Can't generate filename for '{name}' due to s3 key size limit.") + name = os.path.join(dir_name, self.get_alternative_name(file_root, file_ext)) + return name + + def _compress_content(self, content): + """Gzip a given string content.""" + content.seek(0) + zbuf = io.BytesIO() + zfile = GzipFile(mode='wb', fileobj=zbuf, mtime=0.0) + try: + zfile.write(force_bytes(content.read())) + finally: + zfile.close() + zbuf.seek(0) + return zbuf + + def _clean_name(self, name): + clean_name = posixpath.normpath(name).replace('\\', '/') + if name.endswith('/') and not clean_name.endswith('/'): + clean_name += '/' + return clean_name + + def _normalize_name(self, name): + try: + return safe_join(self.base_path, name) + except ValueError: + raise Exception("Attempted access to '%s' denied." % name) + + def _get_write_parameters(self, name, content=None): + params = {} + _type, encoding = mimetypes.guess_type(name) + content_type = getattr(content, 'content_type', None) + content_type = content_type or _type or self.DEFAULT_CONTENT_TYPE + params['ContentType'] = content_type + if encoding: + params['ContentEncoding'] = encoding + return params + + def upload(self, name, content, gzip=True): + if not name: + raise Exception('Name is required') + name = self._normalize_name(self._clean_name(name)) + name = self.get_available_name(name) + params = self._get_write_parameters(name, content) + if ( + gzip and + params['ContentType'] in self.GZIP_CONTENT_TYPES and + 'ContentEncoding' not in params + ): + content = self._compress_content(content) + params['ContentEncoding'] = 'gzip' + obj = self.bucket.Object(name) + content.seek(0, os.SEEK_SET) + obj.upload_fileobj(content, ExtraArgs=params) + return name + + def url(self, name, parameters=None, expire=None, http_method=None): + name = self._normalize_name(self._clean_name(name)) + if expire is None: + expire = self.DEFAULT_QUERYSTRING_EXPIRE + params = parameters.copy() if parameters else {} + params['Bucket'] = self.bucket.name + params['Key'] = name + url = self.bucket.meta.client.generate_presigned_url( + 'get_object', Params=params, ExpiresIn=expire, HttpMethod=http_method + ) + if self.QUERYSTRING_AUTH: + return url + return self._strip_signing_parameters(url) + + def get_file(self, name): + name = self._normalize_name(self._clean_name(name)) + file = SpooledTemporaryFile( + suffix=".S3Boto3StorageFile", + dir='/tmp/' + ) + obj = self.bucket.Object(name) + obj.download_fileobj(file) + file.seek(0) + return file \ No newline at end of file From dc0fd278ce0fd4e43cc2348c342c7857f53eaa07 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 1 Sep 2022 16:34:53 +0545 Subject: [PATCH 22/45] env added; --- modules/ecs/iam_roles.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ecs/iam_roles.tf b/modules/ecs/iam_roles.tf index 206cef9b..bacded08 100644 --- a/modules/ecs/iam_roles.tf +++ b/modules/ecs/iam_roles.tf @@ -23,7 +23,7 @@ resource "aws_iam_role_policy_attachment" "ecs_task_execution_role" { } resource "aws_iam_role" "ecs-custom-tasks" { - name = "ecs-custom-tasks" + name = "ecs-custom-tasks-${var.environment}" assume_role_policy = < Date: Mon, 5 Sep 2022 21:33:24 +0545 Subject: [PATCH 23/45] updated lambda concurrency for staging env; --- main.tf | 6 ++++ modules/reserved_lambda_entry_predict/main.tf | 18 ++++++------ .../reserved_sqs_lambda_extract_docs/main.tf | 28 ++++++++++--------- .../variables.tf | 4 ++- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/main.tf b/main.tf index fad4007f..69a854e8 100644 --- a/main.tf +++ b/main.tf @@ -30,6 +30,10 @@ provider "aws" { # profile = var.aws_profile } +module "secrets_module" { + source = "./modules/secrets" +} + module "sqs_lambda_module" { source = "./modules/sqs_lambda_extract_docs" processed_docs_bucket = "${module.s3_module.te_bucket_name}" @@ -81,6 +85,8 @@ module "reserved_sqs_lambda_module" { environment = var.environment sentry_url = var.sentry_url + + sentry_url_value = "${module.secrets_module.sentry_url_value}" #var.sentry_url } module "sqs_lambda_predict_module" { diff --git a/modules/reserved_lambda_entry_predict/main.tf b/modules/reserved_lambda_entry_predict/main.tf index a8c96fc6..0d1c49d4 100644 --- a/modules/reserved_lambda_entry_predict/main.tf +++ b/modules/reserved_lambda_entry_predict/main.tf @@ -132,7 +132,7 @@ module "reserved_predict_entry_fn" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 10 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 10 reserved_concurrent_executions = 30 environment_variables = { @@ -147,8 +147,8 @@ module "reserved_predict_entry_fn" { } resource "aws_appautoscaling_target" "reserved_predict_entry_fn_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 10 - min_capacity = var.environment == "dev" ? 1 : 5 + max_capacity = var.environment == "staging" ? 1 : 10 + min_capacity = var.environment == "staging" ? 1 : 5 resource_id = "function:${module.reserved_predict_entry_fn.lambda_function_name}:${module.reserved_predict_entry_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -203,7 +203,7 @@ module "reserved_entry_predict_output_fn" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 layers = ["${aws_lambda_layer_version.reserved_lambda_layer_mappings.arn}"] @@ -226,8 +226,8 @@ resource "aws_lambda_layer_version" "reserved_lambda_layer_mappings" { } resource "aws_appautoscaling_target" "reserved_entry_predict_output_fn_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 5 - min_capacity = var.environment == "dev" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 2 resource_id = "function:${module.reserved_entry_predict_output_fn.lambda_function_name}:${module.reserved_entry_predict_output_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -274,7 +274,7 @@ module "reserved_entry_predict_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 environment_variables = { PREDICTION_QUEUE = aws_sqs_queue.reserved_entry_input_processed_queue_predict.id @@ -282,8 +282,8 @@ module "reserved_entry_predict_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_entry_predict_transfer_dlq_msg_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 5 - min_capacity = var.environment == "dev" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 2 resource_id = "function:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_name}:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index ec539935..9e40c14c 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -127,7 +127,8 @@ module "reserved_extract_docs_fn" { "ecs:RunTask", "iam:PassRole", "logs:CreateLogStream", - "logs:PutLogEvents" + "logs:PutLogEvents", + "secretsmanager:*" ], "Resource": [ aws_sqs_queue.reserved_input_queue.arn, @@ -139,13 +140,14 @@ module "reserved_extract_docs_fn" { "arn:aws:lambda:us-east-1:${data.aws_caller_identity.reserved_current_user.account_id}:function:${var.docs_convert_lambda_fn_name}", "${var.ecs_cluster_id}", "${var.ecs_task_definition}", - "arn:aws:iam::${data.aws_caller_identity.reserved_current_user.account_id}:role/*" + "arn:aws:iam::${data.aws_caller_identity.reserved_current_user.account_id}:role/*", + "arn:aws:secretsmanager:us-east-1::secret:*" ] } ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 reserved_concurrent_executions = 30 build_in_docker = true @@ -160,7 +162,7 @@ module "reserved_extract_docs_fn" { ECS_TASK_DEFINITION = "${var.ecs_task_definition}" ECS_CONTAINER_NAME = "${var.ecs_container_name}" ENVIRONMENT = "${var.environment}" - SENTRY_URL = "${var.sentry_url}" + SENTRY_URL = var.sentry_url_value } } @@ -177,8 +179,8 @@ module "reserved_extract_docs_fn" { # } resource "aws_appautoscaling_target" "reserved_extract_docs_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 5 - min_capacity = var.environment == "dev" ? 1 : 3 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 3 resource_id = "function:${module.reserved_extract_docs_fn.lambda_function_name}:${module.reserved_extract_docs_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -225,7 +227,7 @@ module "reserved_output_request_fn" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 build_in_docker = true #store_on_s3 = true @@ -234,13 +236,13 @@ module "reserved_output_request_fn" { environment_variables = { SIGNED_URL_EXPIRY_SECS = "${var.signed_url_expiry_secs}" ENVIRONMENT = "${var.environment}" - SENTRY_URL = "${var.sentry_url}" + SENTRY_URL = "${var.sentry_url_value}" } } resource "aws_appautoscaling_target" "reserved_output_fn_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 5 - min_capacity = var.environment == "dev" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 2 resource_id = "function:${module.reserved_output_request_fn.lambda_function_name}:${module.reserved_output_request_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -288,7 +290,7 @@ module "reserved_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = var.environment == "dev" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 environment_variables = { PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id @@ -296,8 +298,8 @@ module "reserved_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_transfer_dlq_msg_autoscale" { - max_capacity = var.environment == "dev" ? 1 : 5 - min_capacity = var.environment == "dev" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 2 resource_id = "function:${module.reserved_transfer_dlq_msg.lambda_function_name}:${module.reserved_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_sqs_lambda_extract_docs/variables.tf b/modules/reserved_sqs_lambda_extract_docs/variables.tf index 42471848..06a96eb1 100644 --- a/modules/reserved_sqs_lambda_extract_docs/variables.tf +++ b/modules/reserved_sqs_lambda_extract_docs/variables.tf @@ -25,4 +25,6 @@ variable ecs_container_name {} variable docs_convert_bucket_name {} -variable sentry_url {} \ No newline at end of file +variable sentry_url {} + +variable sentry_url_value {} \ No newline at end of file From 4b823a1e49e4444ade77898af0f865beae576600 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Mon, 5 Sep 2022 21:35:25 +0545 Subject: [PATCH 24/45] sentry url fetched from secrets manager; --- main.tf | 11 +++++------ modules/reserved_sqs_lambda_extract_docs/main.tf | 4 ++-- modules/reserved_sqs_lambda_extract_docs/variables.tf | 4 +--- modules/secrets/main.tf | 11 +++++++++++ modules/secrets/output.tf | 3 +++ modules/secrets/variables.tf | 1 + staging.tfvars | 2 +- variables.tf | 2 +- 8 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 modules/secrets/main.tf create mode 100644 modules/secrets/output.tf create mode 100644 modules/secrets/variables.tf diff --git a/main.tf b/main.tf index 69a854e8..5142d722 100644 --- a/main.tf +++ b/main.tf @@ -32,6 +32,7 @@ provider "aws" { module "secrets_module" { source = "./modules/secrets" + sentry_secret_name = var.sentry_secret_name } module "sqs_lambda_module" { @@ -59,7 +60,7 @@ module "sqs_lambda_module" { environment = var.environment - sentry_url = var.sentry_url + sentry_url = "${module.secrets_module.sentry_url_value}" } module "reserved_sqs_lambda_module" { @@ -84,9 +85,7 @@ module "reserved_sqs_lambda_module" { environment = var.environment - sentry_url = var.sentry_url - - sentry_url_value = "${module.secrets_module.sentry_url_value}" #var.sentry_url + sentry_url = "${module.secrets_module.sentry_url_value}" } module "sqs_lambda_predict_module" { @@ -104,7 +103,7 @@ module "sqs_lambda_predict_module" { aws_region = var.aws_region environment = var.environment - sentry_url = var.sentry_url + sentry_url = "${module.secrets_module.sentry_url_value}" } module "reserved_sqs_lambda_predict_module" { @@ -119,7 +118,7 @@ module "reserved_sqs_lambda_predict_module" { aws_region = var.aws_region environment = var.environment - sentry_url = var.sentry_url + sentry_url = "${module.secrets_module.sentry_url_value}" } module "apigateway_module" { diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index 9e40c14c..6b7d9422 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -162,7 +162,7 @@ module "reserved_extract_docs_fn" { ECS_TASK_DEFINITION = "${var.ecs_task_definition}" ECS_CONTAINER_NAME = "${var.ecs_container_name}" ENVIRONMENT = "${var.environment}" - SENTRY_URL = var.sentry_url_value + SENTRY_URL = "${var.sentry_url}" } } @@ -236,7 +236,7 @@ module "reserved_output_request_fn" { environment_variables = { SIGNED_URL_EXPIRY_SECS = "${var.signed_url_expiry_secs}" ENVIRONMENT = "${var.environment}" - SENTRY_URL = "${var.sentry_url_value}" + SENTRY_URL = "${var.sentry_url}" } } diff --git a/modules/reserved_sqs_lambda_extract_docs/variables.tf b/modules/reserved_sqs_lambda_extract_docs/variables.tf index 06a96eb1..42471848 100644 --- a/modules/reserved_sqs_lambda_extract_docs/variables.tf +++ b/modules/reserved_sqs_lambda_extract_docs/variables.tf @@ -25,6 +25,4 @@ variable ecs_container_name {} variable docs_convert_bucket_name {} -variable sentry_url {} - -variable sentry_url_value {} \ No newline at end of file +variable sentry_url {} \ No newline at end of file diff --git a/modules/secrets/main.tf b/modules/secrets/main.tf new file mode 100644 index 00000000..2f31bdfb --- /dev/null +++ b/modules/secrets/main.tf @@ -0,0 +1,11 @@ +data "aws_secretsmanager_secret" "sentry" { + name = var.sentry_secret_name +} + +data "aws_secretsmanager_secret_version" "latest_ver" { + secret_id = data.aws_secretsmanager_secret.sentry.id +} + +data "external" "sentry_json" { + program = ["echo", "${data.aws_secretsmanager_secret_version.latest_ver.secret_string}"] +} \ No newline at end of file diff --git a/modules/secrets/output.tf b/modules/secrets/output.tf new file mode 100644 index 00000000..56da23f4 --- /dev/null +++ b/modules/secrets/output.tf @@ -0,0 +1,3 @@ +output sentry_url_value { + value = data.external.sentry_json.result.sentry_url +} \ No newline at end of file diff --git a/modules/secrets/variables.tf b/modules/secrets/variables.tf new file mode 100644 index 00000000..0076716a --- /dev/null +++ b/modules/secrets/variables.tf @@ -0,0 +1 @@ +variable sentry_secret_name {} \ No newline at end of file diff --git a/staging.tfvars b/staging.tfvars index 6cd4d7a6..69aae20d 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -23,7 +23,7 @@ docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" docs_convert_bucket_name = "deep-large-docs-conversion" # sentry url -sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" +sentry_secret_name = "staging/sentry_url" # VPC az_count = 1 diff --git a/variables.tf b/variables.tf index f2cda84c..04144be0 100644 --- a/variables.tf +++ b/variables.tf @@ -24,7 +24,7 @@ variable docs_convert_lambda_fn_name {} variable docs_convert_bucket_name {} -variable sentry_url {} +variable sentry_secret_name {} variable az_count {} From 67e332443cf3953a8631f0b03093965710cf009a Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Mon, 5 Sep 2022 21:48:55 +0545 Subject: [PATCH 25/45] sentry secret name assign; --- prod.tfvars | 2 +- staging.tfvars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prod.tfvars b/prod.tfvars index df7ae5ea..3d95bd43 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -23,7 +23,7 @@ docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" docs_convert_bucket_name = "deep-large-docs-conversion" # sentry url -sentry_url = "https://3b273f4c61ac4d94af28e85a66ea0b5a@o158798.ingest.sentry.io/1223576" +sentry_secret_name = "prod/sentry_url" # get the name from secrets manager # VPC az_count = 1 diff --git a/staging.tfvars b/staging.tfvars index 69aae20d..5337424e 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -23,7 +23,7 @@ docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" docs_convert_bucket_name = "deep-large-docs-conversion" # sentry url -sentry_secret_name = "staging/sentry_url" +sentry_secret_name = "staging/sentry_url" # get the name from secrets manager # VPC az_count = 1 From 4dd5d457740916ab398f26f5fb02aac59ddf0a68 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Mon, 10 Oct 2022 16:57:05 +0545 Subject: [PATCH 26/45] updates to use cpu model for prediction; --- lambda_fns/entry_predict/app.py | 27 +- .../postprocess_cpu_model_outputs.py | 243 ++++++++++++++++++ .../entry_predict_output_request/app.py | 12 +- modules/reserved_lambda_entry_predict/main.tf | 6 +- staging.tfvars | 6 +- 5 files changed, 262 insertions(+), 32 deletions(-) create mode 100644 lambda_fns/entry_predict/postprocess_cpu_model_outputs.py diff --git a/lambda_fns/entry_predict/app.py b/lambda_fns/entry_predict/app.py index e4d9da91..6f0a8690 100644 --- a/lambda_fns/entry_predict/app.py +++ b/lambda_fns/entry_predict/app.py @@ -8,7 +8,7 @@ import sentry_sdk -from postprocess_raw_preds import get_predictions_all, get_clean_thresholds, get_clean_ratios +from postprocess_cpu_model_outputs import output_all_preds logging.getLogger().setLevel(logging.INFO) @@ -151,27 +151,14 @@ def get_model_info(): def get_predictions(entry): - data = { - "columns": ["excerpt", "return_type"], - "index": [0], - "data": [[entry, "all_models"]] - } - try: - response = sagemaker_rt.invoke_endpoint( - EndpointName=MODEL_ENDPOINT_NAME, - ContentType="application/json; format=pandas-split", - Body=json.dumps(data) - ) - pred_response = json.loads(response["Body"].read().decode("ascii")) - pred_tags = get_clean_ratios(pred_response['raw_predictions']) - pred_thresholds = get_clean_thresholds(pred_response['thresholds']) - tags_selected = get_predictions_all(pred_response['raw_predictions']) + pred_tags, pred_thresholds, tags_selected, error_status = output_all_preds( + entry, + MODEL_ENDPOINT_NAME + ) + if not error_status: prediction_status = PredictionStatus.SUCCESS.value - except ClientError as error: - logging.error(f"Error occurred while getting predictions: {error}") - pred_response = None + else: prediction_status = PredictionStatus.FAILED.value - return pred_tags, pred_thresholds, tags_selected, str(prediction_status) diff --git a/lambda_fns/entry_predict/postprocess_cpu_model_outputs.py b/lambda_fns/entry_predict/postprocess_cpu_model_outputs.py new file mode 100644 index 00000000..c979ecb7 --- /dev/null +++ b/lambda_fns/entry_predict/postprocess_cpu_model_outputs.py @@ -0,0 +1,243 @@ +import json +import boto3 +import logging + +from botocore.exceptions import ClientError +from ast import literal_eval +from typing import List, Dict +from collections import defaultdict + + +logging.getLogger().setLevel(logging.INFO) + +pillars_1d_tags = [ + "Covid-19", + "Casualties", + "Context", + "Displacement", + "Humanitarian Access", + "Shock/Event", + "Information And Communication", +] + +pillars_2d_tags = [ + "At Risk", + "Priority Interventions", + "Capacities & Response", + "Humanitarian Conditions", + "Impact", + "Priority Needs", +] + +secondary_tags = [ + "age", + "gender", + "affected_groups", + "specific_needs_groups", + "severity", +] + + +def get_preds_entry( + preds_column: Dict[str, float], + return_at_least_one=True, + ratio_nb=1, + return_only_one=False, +): + if return_only_one: + preds_entry = [ + sub_tag + for sub_tag, ratio in preds_column.items() + if ratio == max(list(preds_column.values())) + ] + else: + preds_entry = [ + sub_tag for sub_tag, ratio in preds_column.items() if ratio > ratio_nb + ] + if return_at_least_one and len(preds_entry) == 0: + preds_entry = [ + sub_tag + for sub_tag, ratio in preds_column.items() + if ratio == max(list(preds_column.values())) + ] + return preds_entry + + +def get_predictions_all( + ratios_entries: List[Dict[str, float]], + pillars_2d=pillars_2d_tags, + pillars_1d=pillars_1d_tags, + ratio_nb: int = 1, +): + + predictions = defaultdict(list) + for ratio_proba_threshold_one_entry in ratios_entries: + returns_sectors = ratio_proba_threshold_one_entry["primary_tags"]["sectors"] + + subpillars_2d_tags = ratio_proba_threshold_one_entry["primary_tags"][ + "subpillars_2d" + ] + subpillars_1d_tags = ratio_proba_threshold_one_entry["primary_tags"][ + "subpillars_1d" + ] + + ratios_sectors_subpillars_2d = list(returns_sectors.values()) + list( + subpillars_2d_tags.values() + ) + + if any([item >= ratio_nb for item in ratios_sectors_subpillars_2d]): + preds_2d = get_preds_entry(subpillars_2d_tags, True, ratio_nb) + preds_sectors = get_preds_entry(returns_sectors, True, ratio_nb) + + else: + preds_2d = [] + preds_sectors = [] + + predictions["sectors"].append(preds_sectors) + predictions["subpillars_2d"].append(preds_2d) + + preds_1d = get_preds_entry(subpillars_1d_tags, False, ratio_nb) + predictions["subpillars_1d"].append(preds_1d) + + returns_sec_tags = ratio_proba_threshold_one_entry["secondary_tags"] + + for secondary_tag in [ + "age", + "gender", + "affected_groups", + "specific_needs_groups", + ]: + preds_one_sec_tag = get_preds_entry( + returns_sec_tags[secondary_tag], False, ratio_nb + ) + + predictions[secondary_tag].append(preds_one_sec_tag) + + severity_tags = returns_sec_tags["severity"] + if any(["Humanitarian Conditions" in item for item in preds_2d]): + preds_severity = get_preds_entry(severity_tags, True, ratio_nb, True) + else: + preds_severity = [] + predictions["severity"].append(preds_severity) + + return predictions + + +def flatten(t: List[List]) -> List: + return [item for sublist in t for item in sublist] + + +def convert_current_dict_to_previous_one( + ratios_one_entry: Dict[str, float] +) -> Dict[str, Dict[str, float]]: + + # prim tags + primary_tags_results = {"sectors": {}, "subpillars_2d": {}, "subpillars_1d": {}} + + # sec tags + secondary_tags_results = { + "age": {}, + "gender": {}, + "affected_groups": {}, + "specific_needs_groups": {}, + "severity": {}, + } + + for tag, number in ratios_one_entry.items(): + tag_levels = tag.split("->") + if "subpillars" == tag_levels[0]: + assert tag_levels[1] in pillars_1d_tags or tag_levels[1] in pillars_2d_tags + + if tag_levels[1] in pillars_1d_tags: + subpillar_name = "subpillars_1d" + else: + subpillar_name = "subpillars_2d" + + primary_tags_results[subpillar_name]["->".join(tag_levels[1:])] = number + + elif "secondary_tags" == tag_levels[0]: + assert tag_levels[1] in secondary_tags + + secondary_tags_results[tag_levels[1]][tag_levels[2]] = number + + else: + if "sectors" == tag_levels[1]: + primary_tags_results["sectors"][tag_levels[2]] = number + + outputs = { + "primary_tags": primary_tags_results, + "secondary_tags": secondary_tags_results, + } + + return outputs + + +def get_endpoint_probas(df, endpoint_name): + client = boto3.session.Session().client( + "sagemaker-runtime", region_name="us-east-1" + ) + + all_outputs = [] + try: + response = client.invoke_endpoint( + EndpointName=endpoint_name, + Body=json.dumps(df), + ContentType="application/json; format=pandas-split", + ) + except ClientError as error: + logging.error(f"Error occurred while getting predictions: {error}") + return [] + all_outputs = response["Body"].read().decode("ascii") + + return all_outputs + +def output_all_preds(entry, endpoint_name): + data = { + "columns": ["excerpt", "return_type"], + "index": [0], + "data": [[entry, "default_analyis"]] + } + + data["columns"].append("analyis_framework_id") + data["columns"].append("return_prediction_labels") + data["columns"].append("interpretability") + data["columns"].append("ratio_interpreted_labels") + data["columns"].append("attribution_type") + data["columns"].append("output_backbone_embeddings") + data["columns"].append("pooling_type") + data["columns"].append("finetuned_task") + data["columns"].append("embeddings_return_type") + + data["data"][0].append("all") + data["data"][0].append(True) + data["data"][0].append(False) + data["data"][0].append(0.5) + data["data"][0].append("Layer DeepLift") + data["data"][0].append(False) + data["data"][0].append("['cls', 'mean_pooling']") + data["data"][0].append("['first_level_tags', 'secondary_tags', 'subpillars']") + data["data"][0].append("array") + + endpoint_outputs = get_endpoint_probas( + df=data, + endpoint_name=endpoint_name + ) + if endpoint_outputs: + eval_batch = literal_eval(endpoint_outputs) + output_ratios = eval_batch["raw_predictions"] + + thresholds = eval_batch["thresholds"] + + clean_thresholds = convert_current_dict_to_previous_one(thresholds) + + clean_outputs = [ + convert_current_dict_to_previous_one(one_entry_preds) + for one_entry_preds in output_ratios + ] + + final_predictions = get_predictions_all(clean_outputs) + + return clean_outputs, clean_thresholds, dict(final_predictions), False + + return [], {}, {}, True + diff --git a/lambda_fns/entry_predict_output_request/app.py b/lambda_fns/entry_predict_output_request/app.py index b2a5f90b..7d13c0a9 100644 --- a/lambda_fns/entry_predict_output_request/app.py +++ b/lambda_fns/entry_predict_output_request/app.py @@ -234,13 +234,13 @@ def check_demo_grp_selected_tag(gender, age): return True if gender in selected_tags["gender"][0] and age in selected_tags["age"][0] else False all_tags_pred = {} - for prim_tags_key, prim_key_val in pred_data["primary_tags"].items(): + for prim_tags_key, prim_key_val in pred_data[0]["primary_tags"].items(): tags = {} if prim_tags_key in categories: category = categories[prim_tags_key][0] tags[category] = {} - for tag_key, tag_val in prim_key_val[0].items(): + for tag_key, tag_val in prim_key_val.items(): tag = mappings[tag_key][0] tags[category][tag] = {} tags[category][tag]["prediction"] = round(tag_val, 15) @@ -248,13 +248,13 @@ def check_demo_grp_selected_tag(gender, age): tags[category][tag]["is_selected"] = check_selected_tag(prim_tags_key, tag_key) all_tags_pred.update(tags) - for sec_tags_key, sec_tags_val in pred_data["secondary_tags"].items(): + for sec_tags_key, sec_tags_val in pred_data[0]["secondary_tags"].items(): tags = {} if sec_tags_key in categories: category = categories[sec_tags_key][0] tags[category] = {} - for tag_key, tag_val in sec_tags_val[0].items(): + for tag_key, tag_val in sec_tags_val.items(): tag = mappings[tag_key][0] tags[category][tag] = {} tags[category][tag]["prediction"] = round(tag_val, 15) @@ -265,8 +265,8 @@ def check_demo_grp_selected_tag(gender, age): demographic_grp_id = categories['demographic_group'][0] tags = {} tags[demographic_grp_id] = {} - for age_key, age_val in pred_data["secondary_tags"]["age"][0].items(): - for gender_key, gender_val in pred_data["secondary_tags"]["gender"][0].items(): + for age_key, age_val in pred_data[0]["secondary_tags"]["age"].items(): + for gender_key, gender_val in pred_data[0]["secondary_tags"]["gender"].items(): demographic_key = f"{gender_key} {age_key}" if demographic_key in mappings: diff --git a/modules/reserved_lambda_entry_predict/main.tf b/modules/reserved_lambda_entry_predict/main.tf index 0d1c49d4..976f1d44 100644 --- a/modules/reserved_lambda_entry_predict/main.tf +++ b/modules/reserved_lambda_entry_predict/main.tf @@ -132,7 +132,7 @@ module "reserved_predict_entry_fn" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 10 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 reserved_concurrent_executions = 30 environment_variables = { @@ -147,8 +147,8 @@ module "reserved_predict_entry_fn" { } resource "aws_appautoscaling_target" "reserved_predict_entry_fn_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 10 - min_capacity = var.environment == "staging" ? 1 : 5 + max_capacity = var.environment == "staging" ? 1 : 5 + min_capacity = var.environment == "staging" ? 1 : 2 resource_id = "function:${module.reserved_predict_entry_fn.lambda_function_name}:${module.reserved_predict_entry_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/staging.tfvars b/staging.tfvars index 5337424e..ded7e1b5 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -8,7 +8,7 @@ vpce_id = "vpce-02c7bb08b571074e1" vpc_id = "vpc-0e65245d5e4c2deaf" # models -model_endpoint_name = "test-all-models-rsh" +model_endpoint_name = "main-model-cpu" geolocation_fn_name = "geolocations" reliability_fn_name = "reliability" model_info_fn_name = "model_info" @@ -33,7 +33,7 @@ cidr_block = "172.16.0.0/16" ecs_task_execution_role = "ECSTaskExecutionRole" # ECS -fargate_cpu = "2048" -fargate_memory = "4096" +fargate_cpu = "1024" +fargate_memory = "2048" app_count = 1 app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser" From f89506aacf1d6530a41177005affb5cc1aa28b36 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 11 Oct 2022 10:00:41 +0545 Subject: [PATCH 27/45] fixes on env vars usages; refactored code; --- lambda_fns/extract_docs/Dockerfile | 2 +- lambda_fns/extract_docs/app.py | 51 ++++++++++++++--------- lambda_fns/extract_docs/config.py | 24 +++++++++++ lambda_fns/extract_docs/deepex_ecs/app.py | 42 ++++++++++++------- 4 files changed, 82 insertions(+), 37 deletions(-) create mode 100644 lambda_fns/extract_docs/config.py diff --git a/lambda_fns/extract_docs/Dockerfile b/lambda_fns/extract_docs/Dockerfile index b13e8b2e..ad197a9a 100644 --- a/lambda_fns/extract_docs/Dockerfile +++ b/lambda_fns/extract_docs/Dockerfile @@ -25,4 +25,4 @@ RUN apt-get update -y \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* -COPY deepex_ecs/app.py content_types.py wget.py s3_handler.py /code/ +COPY deepex_ecs/app.py content_types.py wget.py s3_handler.py config.py /code/ diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 896c9898..1cedaf81 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -19,25 +19,41 @@ try: from content_types import ExtractContentType, UrlTypes + from config import ( + EXTRACTED_FILE_NAME, + AWS_REGION, + DEST_BUCKET_NAME, + PROCESSED_QUEUE_NAME, + DOCS_CONVERT_LAMBDA_FN_NAME, + DOCS_CONVERSION_BUCKET_NAME, + DOMAIN_NAME, + SENTRY_URL, + ENVIRONMENT, + VPC_PRIVATE_SUBNET, + ECS_CLUSTER_ID, + ECS_TASK_DEFINITION, + ECS_CONTAINER_NAME + ) except ImportError: from .content_types import ExtractContentType, UrlTypes + from .config import ( + EXTRACTED_FILE_NAME, + AWS_REGION, + DEST_BUCKET_NAME, + PROCESSED_QUEUE_NAME, + DOCS_CONVERT_LAMBDA_FN_NAME, + DOCS_CONVERSION_BUCKET_NAME, + DOMAIN_NAME, + SENTRY_URL, + ENVIRONMENT, + VPC_PRIVATE_SUBNET, + ECS_CLUSTER_ID, + ECS_TASK_DEFINITION, + ECS_CONTAINER_NAME + ) logging.getLogger().setLevel(logging.INFO) -EXTRACTED_FILE_NAME = "extract_text.txt" -DEFAULT_AWS_REGION = "us-east-1" - -AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) -DEST_BUCKET_NAME = os.environ.get("DEST_S3_BUCKET") -PROCESSED_QUEUE_NAME = os.environ.get("PROCESSED_QUEUE") -DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") -DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") - -DOMAIN_NAME = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") - -SENTRY_URL = os.environ.get("SENTRY_URL") -ENVIRONMENT = os.environ.get("ENVIRONMENT") - s3_client = boto3.client('s3', region_name=AWS_REGION) sqs_client = boto3.client('sqs', region_name=AWS_REGION) lambda_client = boto3.client('lambda', region_name=AWS_REGION) @@ -351,12 +367,7 @@ def run_fargate_task( client_id, url, callback_url -): - VPC_PRIVATE_SUBNET = os.environ.get("VPC_PRIVATE_SUBNET") - ECS_CLUSTER_ID = os.environ.get("ECS_CLUSTER_ID") - ECS_TASK_DEFINITION = os.environ.get("ECS_TASK_DEFINITION") - ECS_CONTAINER_NAME = os.environ.get("ECS_CONTAINER_NAME") - +): ecs_client = boto3.client('ecs', region_name=AWS_REGION) response = ecs_client.run_task( diff --git a/lambda_fns/extract_docs/config.py b/lambda_fns/extract_docs/config.py new file mode 100644 index 00000000..e4257835 --- /dev/null +++ b/lambda_fns/extract_docs/config.py @@ -0,0 +1,24 @@ +import os + +EXTRACTED_FILE_NAME = "extract_text.txt" +DEFAULT_AWS_REGION = "us-east-1" + +AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) +DEST_BUCKET_NAME = os.environ.get("DEST_S3_BUCKET") +PROCESSED_QUEUE_NAME = os.environ.get("PROCESSED_QUEUE") +DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") +DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") + +DOMAIN_NAME = os.environ.get("EXTRACTOR_DOMAIN_NAME", "http://extractor:8001") + +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") + +VPC_PRIVATE_SUBNET = os.environ.get("VPC_PRIVATE_SUBNET") +ECS_CLUSTER_ID = os.environ.get("ECS_CLUSTER_ID") +ECS_TASK_DEFINITION = os.environ.get("ECS_TASK_DEFINITION") +ECS_CONTAINER_NAME = os.environ.get("ECS_CONTAINER_NAME") + +CLIENT_ID = os.environ.get("CLIENT_ID") +URL = os.environ.get("URL") +CALLBACK_URL = os.environ.get("CALLBACK_URL") \ No newline at end of file diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index bd135d9a..57b625f5 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -20,28 +20,38 @@ try: from content_types import ExtractContentType, UrlTypes from s3_handler import Storage + from config import ( + EXTRACTED_FILE_NAME, + AWS_REGION, + DEST_BUCKET_NAME, + PROCESSED_QUEUE_NAME, + DOCS_CONVERT_LAMBDA_FN_NAME, + DOCS_CONVERSION_BUCKET_NAME, + SENTRY_URL, + ENVIRONMENT, + CLIENT_ID, + URL, + CALLBACK_URL + ) except ImportError: from .content_types import ExtractContentType, UrlTypes from .s3_handler import Storage + from .config import ( + EXTRACTED_FILE_NAME, + AWS_REGION, + DEST_BUCKET_NAME, + PROCESSED_QUEUE_NAME, + DOCS_CONVERT_LAMBDA_FN_NAME, + DOCS_CONVERSION_BUCKET_NAME, + SENTRY_URL, + ENVIRONMENT, + CLIENT_ID, + URL, + CALLBACK_URL + ) logging.getLogger().setLevel(logging.INFO) -EXTRACTED_FILE_NAME = "extract_text.txt" -DEFAULT_AWS_REGION = "us-east-1" - -AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) -DEST_BUCKET_NAME = os.environ.get("DEST_S3_BUCKET") -PROCESSED_QUEUE_NAME = os.environ.get("PROCESSED_QUEUE") -DOCS_CONVERT_LAMBDA_FN_NAME = os.environ.get("DOCS_CONVERT_LAMBDA_FN_NAME") -DOCS_CONVERSION_BUCKET_NAME = os.environ.get("DOCS_CONVERSION_BUCKET_NAME") - -SENTRY_URL = os.environ.get("SENTRY_URL") -ENVIRONMENT = os.environ.get("ENVIRONMENT") - -CLIENT_ID = os.environ.get("CLIENT_ID") -URL = os.environ.get("URL") -CALLBACK_URL = os.environ.get("CALLBACK_URL") - s3_client = boto3.client('s3', region_name=AWS_REGION) sqs_client = boto3.client('sqs', region_name=AWS_REGION) lambda_client = boto3.client('lambda', region_name=AWS_REGION) From a3b9a04de53e8a354e40791ed87cb47db0c1a45b Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 11 Oct 2022 10:25:43 +0545 Subject: [PATCH 28/45] removed unused dependency; --- lambda_fns/extract_docs/app.py | 1 - lambda_fns/extract_docs/content_types.py | 1 - lambda_fns/extract_docs/deepex_ecs/app.py | 1 - 3 files changed, 3 deletions(-) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 1cedaf81..81e10b3c 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -1,4 +1,3 @@ -import os import requests import uuid import json diff --git a/lambda_fns/extract_docs/content_types.py b/lambda_fns/extract_docs/content_types.py index ad3cd739..862f3a14 100644 --- a/lambda_fns/extract_docs/content_types.py +++ b/lambda_fns/extract_docs/content_types.py @@ -1,4 +1,3 @@ -import os import requests from enum import Enum import logging diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index 57b625f5..23af7731 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -1,4 +1,3 @@ -import os import requests import uuid import json From 50242436475399bcf76f3c5eccc6311bb8781a4e Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 11 Oct 2022 14:27:27 +0545 Subject: [PATCH 29/45] updated env vars --- lambda_fns/extract_docs_output_request/app.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lambda_fns/extract_docs_output_request/app.py b/lambda_fns/extract_docs_output_request/app.py index 8ba82606..d083a778 100644 --- a/lambda_fns/extract_docs_output_request/app.py +++ b/lambda_fns/extract_docs_output_request/app.py @@ -13,8 +13,8 @@ REQUEST_TIMEOUT = 60 DEFAULT_AWS_REGION = "us-east-1" -aws_region = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) -signed_url_expiry_secs = os.environ.get("SIGNED_URL_EXPIRY_SECS") +AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) +SIGNED_URL_EXPIRY_SECS = os.environ.get("SIGNED_URL_EXPIRY_SECS") SENTRY_URL = os.environ.get("SENTRY_URL") ENVIRONMENT = os.environ.get("ENVIRONMENT") @@ -23,13 +23,13 @@ s3_client = boto3.client( 's3', - region_name=aws_region, + region_name=AWS_REGION, config=Config( signature_version='s3v4', s3={'addressing_style': 'path'} ) ) -sqs_client = boto3.client('sqs', region_name=aws_region) +sqs_client = boto3.client('sqs', region_name=AWS_REGION) def extract_path(filepath): @@ -54,7 +54,7 @@ def generate_signed_url(bucket_name, key_name): 'Bucket': bucket_name, 'Key': key_name }, - ExpiresIn=signed_url_expiry_secs + ExpiresIn=SIGNED_URL_EXPIRY_SECS ) except ClientError as e: logging.error(f"ClientError: {e}") From b6daeb3b802247a5df248d7742e380d3939728f3 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 11 Oct 2022 14:28:48 +0545 Subject: [PATCH 30/45] moved env vars to a separate file; --- lambda_fns/entry_predict/app.py | 40 ++++++++++++++---------------- lambda_fns/entry_predict/config.py | 13 ++++++++++ 2 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 lambda_fns/entry_predict/config.py diff --git a/lambda_fns/entry_predict/app.py b/lambda_fns/entry_predict/app.py index 6f0a8690..39251ce3 100644 --- a/lambda_fns/entry_predict/app.py +++ b/lambda_fns/entry_predict/app.py @@ -1,34 +1,32 @@ -import os import boto3 -from botocore.exceptions import ClientError import json -from enum import Enum -import logging -from concurrent.futures import ThreadPoolExecutor, as_completed - import sentry_sdk +import logging +from enum import Enum +from botocore.exceptions import ClientError +from concurrent.futures import ThreadPoolExecutor, as_completed from postprocess_cpu_model_outputs import output_all_preds -logging.getLogger().setLevel(logging.INFO) - -DEFAULT_AWS_REGION = "us-east-1" - -AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) -PREDICTION_QUEUE_NAME = os.environ.get("PREDICTION_QUEUE") -MODEL_ENDPOINT_NAME = os.environ.get("MODEL_ENDPOINT_NAME") -GEOLOCATION_FN_NAME = os.environ.get("GEOLOCATION_FN_NAME") -RELIABILITY_FN_NAME = os.environ.get("RELIABILITY_FN_NAME") -MODEL_INFO_FN_NAME = os.environ.get("MODEL_INFO_FN_NAME") +from config import ( + DEFAULT_AWS_REGION, + AWS_REGION, + PREDICTION_QUEUE_NAME, + MODEL_ENDPOINT_NAME, + GEOLOCATION_FN_NAME, + RELIABILITY_FN_NAME, + MODEL_INFO_FN_NAME, + SENTRY_URL, + ENVIRONMENT +) -SENTRY_URL = os.environ.get("SENTRY_URL") -ENVIRONMENT = os.environ.get("ENVIRONMENT") +logging.getLogger().setLevel(logging.INFO) sqs_client = boto3.client('sqs', region_name=AWS_REGION) -sagemaker_rt = boto3.client("runtime.sagemaker", region_name="us-east-1") # todo: update the region later. -geolocation_client = boto3.client("lambda", region_name="us-east-1") -reliability_client = boto3.client("lambda", region_name="us-east-1") +sagemaker_rt = boto3.client("runtime.sagemaker", region_name=DEFAULT_AWS_REGION) +geolocation_client = boto3.client("lambda", region_name=DEFAULT_AWS_REGION) +reliability_client = boto3.client("lambda", region_name=DEFAULT_AWS_REGION) model_info_client = boto3.client("lambda", region_name=AWS_REGION) sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) diff --git a/lambda_fns/entry_predict/config.py b/lambda_fns/entry_predict/config.py new file mode 100644 index 00000000..0a429adc --- /dev/null +++ b/lambda_fns/entry_predict/config.py @@ -0,0 +1,13 @@ +import os + +DEFAULT_AWS_REGION = "us-east-1" + +AWS_REGION = os.environ.get("AWS_REGION", DEFAULT_AWS_REGION) +PREDICTION_QUEUE_NAME = os.environ.get("PREDICTION_QUEUE") +MODEL_ENDPOINT_NAME = os.environ.get("MODEL_ENDPOINT_NAME") +GEOLOCATION_FN_NAME = os.environ.get("GEOLOCATION_FN_NAME") +RELIABILITY_FN_NAME = os.environ.get("RELIABILITY_FN_NAME") +MODEL_INFO_FN_NAME = os.environ.get("MODEL_INFO_FN_NAME") + +SENTRY_URL = os.environ.get("SENTRY_URL") +ENVIRONMENT = os.environ.get("ENVIRONMENT") \ No newline at end of file From da2e6da3f42674994940813a7fbcda1bbe1986ff Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 13 Oct 2022 11:48:30 +0545 Subject: [PATCH 31/45] prod vars updated; --- prod.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prod.tfvars b/prod.tfvars index 3d95bd43..706c2cef 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -8,7 +8,7 @@ vpce_id = "vpce-05d8c268ef4c0c443" vpc_id = "vpc-0947f040a9d4692a7" # models -model_endpoint_name = "test-all-models-rsh" +model_endpoint_name = "main-model-cpu" geolocation_fn_name = "geolocations" reliability_fn_name = "reliability" model_info_fn_name = "model_info" @@ -20,7 +20,7 @@ docs_extract_fn_image_name = "extract-tool" docs_convert_lambda_fn_name = "libreoffice-dev-libreoffice" # docs convert bucket name -docs_convert_bucket_name = "deep-large-docs-conversion" +docs_convert_bucket_name = "deep-large-docs-conversion-prod" # sentry url sentry_secret_name = "prod/sentry_url" # get the name from secrets manager From b27963f84ffd2762049e3c2126df0387aae6f393 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 13 Dec 2022 20:32:08 +0545 Subject: [PATCH 32/45] parsing lib updated; method updated; --- lambda_fns/extract_docs/deepex_ecs/app.py | 4 +- lambda_fns/extract_docs/poetry.lock | 347 +++------------------- lambda_fns/extract_docs/pyproject.toml | 2 +- 3 files changed, 47 insertions(+), 306 deletions(-) diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index 23af7731..4d17c43a 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -224,7 +224,7 @@ def get_extracted_text(file_path, file_name): binary = base64.b64encode(f.read()) document = TextFromFile(stream=binary, ext="pdf") - entries, images = document.serial_extract_text(output_format="list") + entries, images = document.extract_text(output_format="list") except Exception as e: logging.error(f"Extraction failed: {str(e)}", exc_info=True) return None, None, -1, -1, 0 @@ -274,7 +274,7 @@ def get_extracted_text(file_path, file_name): def get_extracted_text_html_links(link, file_name): try: web_text = TextFromWeb(url=link) - entries = web_text.extract_text(output_format="list") + entries = web_text.extract_text(output_format="list", url=link) except Exception as e: logging.error(f"Extraction from website failed {e}", exc_info=True) return None, None, -1, -1, 0 diff --git a/lambda_fns/extract_docs/poetry.lock b/lambda_fns/extract_docs/poetry.lock index cd19a4aa..f117bfc4 100644 --- a/lambda_fns/extract_docs/poetry.lock +++ b/lambda_fns/extract_docs/poetry.lock @@ -25,8 +25,8 @@ python-versions = ">=3.6.0" soupsieve = ">1.2" [package.extras] -html5lib = ["html5lib"] lxml = ["lxml"] +html5lib = ["html5lib"] [[package]] name = "boto3" @@ -92,15 +92,15 @@ unicode_backport = ["unicodedata2"] [[package]] name = "cssselect" -version = "1.1.0" +version = "1.2.0" description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.7" [[package]] name = "deep-parser" -version = "0.1.0" +version = "0.2.0" description = "This package contains parsing utilities for text extraction" category = "main" optional = false @@ -123,12 +123,12 @@ requests-html = "*" type = "git" url = "https://github.com/the-deep/deepex.git" reference = "master" -resolved_reference = "cee0daf673f1e1ead2a734880130a2e1b09e7fd6" +resolved_reference = "360766f8d51dc26e4b507e449663dbca1be87e38" [[package]] name = "fake-useragent" -version = "0.1.11" -description = "Up to date simple useragent faker with real world database" +version = "1.1.1" +description = "Up-to-date simple useragent faker with real world database" category = "main" optional = false python-versions = "*" @@ -166,10 +166,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" [package.extras] -cssselect = ["cssselect (>=0.7)"] -html5 = ["html5lib"] -htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] +htmlsoup = ["beautifulsoup4"] +html5 = ["html5lib"] +cssselect = ["cssselect (>=0.7)"] [[package]] name = "numpy" @@ -366,7 +366,7 @@ python-versions = ">=3.6" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -387,18 +387,15 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "w3lib" -version = "1.22.0" +version = "2.1.1" description = "Library of web-related functions" category = "main" optional = false -python-versions = "*" - -[package.dependencies] -six = ">=1.4.1" +python-versions = ">=3.7" [[package]] name = "websockets" -version = "10.3" +version = "10.4" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" category = "main" optional = false @@ -407,299 +404,43 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = ">=3.10, <3.11" -content-hash = "6f9776c669bb795a23828b4f2c7cb765a4901712de650f6ceef0f10fd1778f65" +content-hash = "9d20a92ad2e27e42e64b062363f9cee3b36553d422b8994d81b7b28456d30ad4" [metadata.files] -anytree = [ - {file = "anytree-2.8.0-py2.py3-none-any.whl", hash = "sha256:14c55ac77492b11532395049a03b773d14c7e30b22aa012e337b1e983de31521"}, - {file = "anytree-2.8.0.tar.gz", hash = "sha256:3f0f93f355a91bc3e6245319bf4c1d50e3416cc7a35cc1133c1ff38306bbccab"}, -] -beautifulsoup4 = [ - {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, - {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, -] -boto3 = [ - {file = "boto3-1.19.0-py3-none-any.whl", hash = "sha256:d468b1f63f22ccd6b4bfbdebe6fd0c0b4620f38276af965ed139fe3eb85d16bb"}, - {file = "boto3-1.19.0.tar.gz", hash = "sha256:f93fed6153f7def66f1b17e6794c6ec3bec46229b213d3fa63f1eca126f5e992"}, -] -botocore = [ - {file = "botocore-1.22.12-py3-none-any.whl", hash = "sha256:1d1094fb53ebe4535d8840fbd7c14aadb65bde7ff03a65f9a4f1d76bd03e16ff"}, - {file = "botocore-1.22.12.tar.gz", hash = "sha256:fc59b55e8c5dde64b017b2f114c25f8cce397b667e812aea7eafb4b59b49d7cb"}, -] -bs4 = [ - {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"}, -] -certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, - {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, -] -cssselect = [ - {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, - {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"}, -] +anytree = [] +beautifulsoup4 = [] +boto3 = [] +botocore = [] +bs4 = [] +certifi = [] +charset-normalizer = [] +cssselect = [] deep-parser = [] -fake-useragent = [ - {file = "fake-useragent-0.1.11.tar.gz", hash = "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"}, -] -func-timeout = [ - {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"}, -] -idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, -] -jmespath = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] +fake-useragent = [] +func-timeout = [] +idna = [] +jmespath = [] lxml = [] -numpy = [ - {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13"}, - {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218"}, - {file = "numpy-1.21.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1"}, - {file = "numpy-1.21.2-cp37-cp37m-win32.whl", hash = "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310"}, - {file = "numpy-1.21.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f"}, - {file = "numpy-1.21.2-cp38-cp38-win32.whl", hash = "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557"}, - {file = "numpy-1.21.2-cp38-cp38-win_amd64.whl", hash = "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f"}, - {file = "numpy-1.21.2-cp39-cp39-win32.whl", hash = "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad"}, - {file = "numpy-1.21.2-cp39-cp39-win_amd64.whl", hash = "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724"}, - {file = "numpy-1.21.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7"}, - {file = "numpy-1.21.2.zip", hash = "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc"}, -] -parse = [ - {file = "parse-1.19.0.tar.gz", hash = "sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b"}, -] -pillow = [ - {file = "Pillow-8.3.2-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4"}, - {file = "Pillow-8.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a"}, - {file = "Pillow-8.3.2-cp310-cp310-win32.whl", hash = "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228"}, - {file = "Pillow-8.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875"}, - {file = "Pillow-8.3.2-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550"}, - {file = "Pillow-8.3.2-cp36-cp36m-win32.whl", hash = "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073"}, - {file = "Pillow-8.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196"}, - {file = "Pillow-8.3.2-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da"}, - {file = "Pillow-8.3.2-cp37-cp37m-win32.whl", hash = "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9"}, - {file = "Pillow-8.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3"}, - {file = "Pillow-8.3.2-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616"}, - {file = "Pillow-8.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b"}, - {file = "Pillow-8.3.2-cp38-cp38-win32.whl", hash = "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341"}, - {file = "Pillow-8.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb"}, - {file = "Pillow-8.3.2-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9"}, - {file = "Pillow-8.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441"}, - {file = "Pillow-8.3.2-cp39-cp39-win32.whl", hash = "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09"}, - {file = "Pillow-8.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96"}, - {file = "Pillow-8.3.2.tar.gz", hash = "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c"}, -] -pyee = [ - {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"}, - {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, -] -pymupdf = [ - {file = "PyMuPDF-1.18.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4546e426e00e3ace148adaf95e0952061391ea04ec285c7a4592e11b9a372d0b"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0dac1bc4c474dce4d0569b02eb11126f15d9a12d4855fddf21471cff75db493e"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-win32.whl", hash = "sha256:8e7d1036a135ecbe30d185d968556367b018c9954043e17408f1798c1d0a7685"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-win_amd64.whl", hash = "sha256:87f5af9570e6b12191b05e23dfd3e1fd8940f2c0a1ec0773c6496205fdf24f7c"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3c7e01284d9832c3ba76f6eb5c6825827f26749eb20cfe20f633bf980b3d3d85"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ed93f0964b49033073704e4bfb0908046c799731ee1e357f2b4ec934e551b515"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c76115e81a8a1596fe127b9b8b45ec846cb6bcfa891460a2efad4eb145573a4"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-win32.whl", hash = "sha256:615ed1e4d13d404e219c4caff0c8c7e5557a331ffaeb162e12016562ff160178"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-win_amd64.whl", hash = "sha256:3e805320eb1c6d4957d66b06baa4c2726fd0bb84acd5114be08a69c0a071abe6"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d0b1a211bea5f98556dec57172675958e28a90818d44dc6b095efb37034854b8"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57a8c34a236e7021f6defda501eb2afa310687fd45cfb650211d09e9ed4252e7"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecacfef4760c8b451284940d5dbd9aeeb36fd0c1e495545c4f487196047e167d"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-win32.whl", hash = "sha256:20591b790035f5d0cebfcb0d12f336e7d2bab1d708d20a20c8a75e8c6bb51481"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-win_amd64.whl", hash = "sha256:a89a528e3218026ffb8ccceffaa813b4a0bb2e4b8edb3a44371b0f697b18349c"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9212f95bb2978ddf3669dd8feef09ea47c13603b58efee6a305b6fe4a8a6a621"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3c605f2f2936067f9e2ce7d76127bb040c4528a347222ae23d7f440af8965afd"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a21735d8306d058adb62dcdd71d70f32c614f23729abc3971c1eb9099712069c"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-win32.whl", hash = "sha256:c385f38bc7d299917c3c64a096447d8fb573373705c120efec316fd109f53de8"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-win_amd64.whl", hash = "sha256:9fca5a5666e3a09c9b573d2edd230a0e51c379d1b9e11f7121c622f5795fcce5"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e4581e2bbee1392cac203022e93de259221f9dd2ebfd1ca7e2ac1f00d454cff"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ee986ff5ed40b633b7e4bf321658ab31f1285b01628e2830c4bddf0003677f36"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ac77aac8db26dfb118dcc1b5f7857f7b669e9a415a76cc2fb77fd8429af2d4f"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-win32.whl", hash = "sha256:4379bf78ecf4ba17291ba01582ca02e5ddbbc96004b31e8be1cc1b582795b880"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-win_amd64.whl", hash = "sha256:b7dbdfa078069413b3ef442f2b21770477bb7c1ce724b4b332198ca7bc3efffe"}, - {file = "PyMuPDF-1.18.17.tar.gz", hash = "sha256:fa39ee5e91eae77818e07b6bb7e0cb0b402ad88e39a74b08626ce1c2150c5414"}, -] -pyppeteer = [ - {file = "pyppeteer-0.0.14.tar.gz", hash = "sha256:4e0409fb30bb717296432d5548f6a3407d78d9efcf7a17f308fdb42d43607d9f"}, -] -pyquery = [ - {file = "pyquery-1.4.3-py3-none-any.whl", hash = "sha256:1fc33b7699455ed25c75282bc8f80ace1ac078b0dda5a933dacbd8b1c1f83963"}, - {file = "pyquery-1.4.3.tar.gz", hash = "sha256:a388eefb6bc4a55350de0316fbd97cda999ae669b6743ae5b99102ba54f5aa72"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pyyaml = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, -] -readable-content = [ - {file = "readable-content-0.1.2.tar.gz", hash = "sha256:94e6c41c95db814473a190170c861d7a2a20ce04b245c2ecbd20da9eef1f8f93"}, -] -requests = [ - {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, - {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, -] -requests-html = [ - {file = "requests-html-0.10.0.tar.gz", hash = "sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947"}, - {file = "requests_html-0.10.0-py3-none-any.whl", hash = "sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2"}, -] -s3transfer = [ - {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"}, - {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"}, -] +numpy = [] +parse = [] +pillow = [] +pyee = [] +pymupdf = [] +pyppeteer = [] +pyquery = [] +python-dateutil = [] +pyyaml = [] +readable-content = [] +requests = [] +requests-html = [] +s3transfer = [] sentry-sdk = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -soupsieve = [ - {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, - {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, -] +soupsieve = [] typing-extensions = [] -urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, -] -w3lib = [ - {file = "w3lib-1.22.0-py2.py3-none-any.whl", hash = "sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53"}, - {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, -] -websockets = [ - {file = "websockets-10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978"}, - {file = "websockets-10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500"}, - {file = "websockets-10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47"}, - {file = "websockets-10.3-cp310-cp310-win32.whl", hash = "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae"}, - {file = "websockets-10.3-cp310-cp310-win_amd64.whl", hash = "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079"}, - {file = "websockets-10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6"}, - {file = "websockets-10.3-cp37-cp37m-win32.whl", hash = "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1"}, - {file = "websockets-10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4"}, - {file = "websockets-10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36"}, - {file = "websockets-10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69"}, - {file = "websockets-10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76"}, - {file = "websockets-10.3-cp38-cp38-win32.whl", hash = "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559"}, - {file = "websockets-10.3-cp38-cp38-win_amd64.whl", hash = "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d"}, - {file = "websockets-10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094"}, - {file = "websockets-10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667"}, - {file = "websockets-10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8"}, - {file = "websockets-10.3-cp39-cp39-win32.whl", hash = "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582"}, - {file = "websockets-10.3-cp39-cp39-win_amd64.whl", hash = "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02"}, - {file = "websockets-10.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755"}, - {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, - {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, -] +urllib3 = [] +w3lib = [] +websockets = [] diff --git a/lambda_fns/extract_docs/pyproject.toml b/lambda_fns/extract_docs/pyproject.toml index 34a23fc0..4a217930 100644 --- a/lambda_fns/extract_docs/pyproject.toml +++ b/lambda_fns/extract_docs/pyproject.toml @@ -17,7 +17,7 @@ requests = "==2.26.0" urllib3 = "==1.26.6" Pillow = "==8.3.2" boto3 = "==1.19.0" -deep-parser = { git = "https://github.com/the-deep/deepex.git", branch = "master" } +deep-parser = { git = "https://github.com/the-deep/deepex.git", rev="360766f8d51dc26e4b507e449663dbca1be87e38", branch = "master" } sentry-sdk = "==1.5.8" [tool.poetry.dev-dependencies] From 46532b2ab8fa1a3e98557b7926d5541c89532abd Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 13 Dec 2022 20:33:00 +0545 Subject: [PATCH 33/45] tf vars updated on ecr url; --- prod.tfvars | 2 +- staging.tfvars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prod.tfvars b/prod.tfvars index 706c2cef..324d4d2d 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -36,4 +36,4 @@ ecs_task_execution_role = "ECSTaskExecutionRole" fargate_cpu = "2048" fargate_memory = "4096" app_count = 1 -app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser" \ No newline at end of file +app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser-prod" \ No newline at end of file diff --git a/staging.tfvars b/staging.tfvars index ded7e1b5..6feaa47f 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -36,4 +36,4 @@ ecs_task_execution_role = "ECSTaskExecutionRole" fargate_cpu = "1024" fargate_memory = "2048" app_count = 1 -app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser" +app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser-staging" From 80fe746c7a6b3b3832afc03a1f90dc64798323e1 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Tue, 13 Dec 2022 20:34:11 +0545 Subject: [PATCH 34/45] utf-8 added to content type while saving file in s3; --- lambda_fns/extract_docs/s3_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda_fns/extract_docs/s3_handler.py b/lambda_fns/extract_docs/s3_handler.py index 16212188..05e3d21c 100644 --- a/lambda_fns/extract_docs/s3_handler.py +++ b/lambda_fns/extract_docs/s3_handler.py @@ -138,7 +138,7 @@ def _get_write_parameters(self, name, content=None): params = {} _type, encoding = mimetypes.guess_type(name) content_type = getattr(content, 'content_type', None) - content_type = content_type or _type or self.DEFAULT_CONTENT_TYPE + content_type = "text/plain; charset=utf-8" #content_type or _type or self.DEFAULT_CONTENT_TYPE params['ContentType'] = content_type if encoding: params['ContentEncoding'] = encoding From 1b1f94bbd441877f8e65810e877c35ec9f0ddc16 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 23 Dec 2022 11:58:52 +0545 Subject: [PATCH 35/45] added vpc endpoint for transferring data to s3; --- modules/ecs/vpc.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/ecs/vpc.tf b/modules/ecs/vpc.tf index 577599d8..2c279d5c 100644 --- a/modules/ecs/vpc.tf +++ b/modules/ecs/vpc.tf @@ -85,4 +85,15 @@ resource "aws_route_table_association" "private" { count = var.az_count subnet_id = element(aws_subnet.private.*.id, count.index) route_table_id = element(aws_route_table.private.*.id, count.index) +} + +# Create a VPC endpoint for s3 +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.vpc.id + service_name = "com.amazonaws.${var.aws_region}.s3" + route_table_ids = ["${aws_route_table.private[0].id}"] + + tags = { + Name = "s3-endpoint-${var.environment}" + } } \ No newline at end of file From 86f2f8ee82789491c07e0ef8c55dc33cbf911e62 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 23 Dec 2022 12:00:54 +0545 Subject: [PATCH 36/45] added variables for setting the lambda concurrency --- modules/reserved_lambda_entry_predict/main.tf | 18 +++++++++--------- .../reserved_lambda_entry_predict/variables.tf | 10 +++++++++- .../reserved_sqs_lambda_extract_docs/main.tf | 18 +++++++++--------- .../variables.tf | 10 +++++++++- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/modules/reserved_lambda_entry_predict/main.tf b/modules/reserved_lambda_entry_predict/main.tf index 976f1d44..60d5ee8c 100644 --- a/modules/reserved_lambda_entry_predict/main.tf +++ b/modules/reserved_lambda_entry_predict/main.tf @@ -132,7 +132,7 @@ module "reserved_predict_entry_fn" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max reserved_concurrent_executions = 30 environment_variables = { @@ -147,8 +147,8 @@ module "reserved_predict_entry_fn" { } resource "aws_appautoscaling_target" "reserved_predict_entry_fn_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_predict_entry_fn.lambda_function_name}:${module.reserved_predict_entry_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -203,7 +203,7 @@ module "reserved_entry_predict_output_fn" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max layers = ["${aws_lambda_layer_version.reserved_lambda_layer_mappings.arn}"] @@ -226,8 +226,8 @@ resource "aws_lambda_layer_version" "reserved_lambda_layer_mappings" { } resource "aws_appautoscaling_target" "reserved_entry_predict_output_fn_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_entry_predict_output_fn.lambda_function_name}:${module.reserved_entry_predict_output_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -274,7 +274,7 @@ module "reserved_entry_predict_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max environment_variables = { PREDICTION_QUEUE = aws_sqs_queue.reserved_entry_input_processed_queue_predict.id @@ -282,8 +282,8 @@ module "reserved_entry_predict_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_entry_predict_transfer_dlq_msg_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_name}:${module.reserved_entry_predict_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_lambda_entry_predict/variables.tf b/modules/reserved_lambda_entry_predict/variables.tf index d02dcd0b..72e1d2b5 100644 --- a/modules/reserved_lambda_entry_predict/variables.tf +++ b/modules/reserved_lambda_entry_predict/variables.tf @@ -12,4 +12,12 @@ variable model_info_fn_name {} variable processed_docs_bucket {} -variable sentry_url {} \ No newline at end of file +variable sentry_url {} + +variable lambda_concurrency_max { + default = 3 +} + +variable lambda_concurrency_min { + default = 2 +} \ No newline at end of file diff --git a/modules/reserved_sqs_lambda_extract_docs/main.tf b/modules/reserved_sqs_lambda_extract_docs/main.tf index 6b7d9422..85b7af10 100644 --- a/modules/reserved_sqs_lambda_extract_docs/main.tf +++ b/modules/reserved_sqs_lambda_extract_docs/main.tf @@ -147,7 +147,7 @@ module "reserved_extract_docs_fn" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max reserved_concurrent_executions = 30 build_in_docker = true @@ -179,8 +179,8 @@ module "reserved_extract_docs_fn" { # } resource "aws_appautoscaling_target" "reserved_extract_docs_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 3 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_extract_docs_fn.lambda_function_name}:${module.reserved_extract_docs_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -227,7 +227,7 @@ module "reserved_output_request_fn" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max build_in_docker = true #store_on_s3 = true @@ -241,8 +241,8 @@ module "reserved_output_request_fn" { } resource "aws_appautoscaling_target" "reserved_output_fn_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_output_request_fn.lambda_function_name}:${module.reserved_output_request_fn.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" @@ -290,7 +290,7 @@ module "reserved_transfer_dlq_msg" { ] }) - provisioned_concurrent_executions = var.environment == "staging" ? 1 : 5 + provisioned_concurrent_executions = var.environment == "staging" ? 1 : var.lambda_concurrency_max environment_variables = { PROCESSED_QUEUE = aws_sqs_queue.reserved_processed_queue.id @@ -298,8 +298,8 @@ module "reserved_transfer_dlq_msg" { } resource "aws_appautoscaling_target" "reserved_transfer_dlq_msg_autoscale" { - max_capacity = var.environment == "staging" ? 1 : 5 - min_capacity = var.environment == "staging" ? 1 : 2 + max_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_max + min_capacity = var.environment == "staging" ? 1 : var.lambda_concurrency_min resource_id = "function:${module.reserved_transfer_dlq_msg.lambda_function_name}:${module.reserved_transfer_dlq_msg.lambda_function_version}" scalable_dimension = "lambda:function:ProvisionedConcurrency" service_namespace = "lambda" diff --git a/modules/reserved_sqs_lambda_extract_docs/variables.tf b/modules/reserved_sqs_lambda_extract_docs/variables.tf index 42471848..93ba38c3 100644 --- a/modules/reserved_sqs_lambda_extract_docs/variables.tf +++ b/modules/reserved_sqs_lambda_extract_docs/variables.tf @@ -25,4 +25,12 @@ variable ecs_container_name {} variable docs_convert_bucket_name {} -variable sentry_url {} \ No newline at end of file +variable sentry_url {} + +variable lambda_concurrency_max { + default = 3 +} + +variable lambda_concurrency_min { + default = 2 +} \ No newline at end of file From 828bdb1a3e8fde4693aeeaee15ca18a631a792c5 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 4 Jan 2023 11:51:48 +0545 Subject: [PATCH 37/45] updated poetry lock file; added sentry pkg; --- mockserver/poetry.lock | 745 +++++++------------------------------- mockserver/pyproject.toml | 5 +- 2 files changed, 140 insertions(+), 610 deletions(-) diff --git a/mockserver/poetry.lock b/mockserver/poetry.lock index a174e923..095768ac 100644 --- a/mockserver/poetry.lock +++ b/mockserver/poetry.lock @@ -44,8 +44,8 @@ python-versions = ">=3.6.0" soupsieve = ">1.2" [package.extras] -html5lib = ["html5lib"] lxml = ["lxml"] +html5lib = ["html5lib"] [[package]] name = "billiard" @@ -220,23 +220,23 @@ six = "*" [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "cssselect" -version = "1.1.0" +version = "1.2.0" description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.7" [[package]] name = "deep-parser" -version = "0.1.0" +version = "0.2.0" description = "This package contains parsing utilities for text extraction" category = "main" optional = false @@ -259,30 +259,19 @@ requests-html = "*" type = "git" url = "https://github.com/the-deep/deepex.git" reference = "master" -resolved_reference = "68c0d5d3b64abe16aa1313e11479709fc1362a38" - -[[package]] -name = "deprecated" -version = "1.2.13" -description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -wrapt = ">=1.10,<2" - -[package.extras] -dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "importlib-resources (<4)", "configparser (<5)", "sphinxcontrib-websupport (<2)", "zipp (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"] +resolved_reference = "f584626954c71bc2cb530a380b652bd53a2a30d4" [[package]] name = "fake-useragent" -version = "0.1.11" -description = "Up to date simple useragent faker with real world database" +version = "1.1.1" +description = "Up-to-date simple useragent faker with real world database" category = "main" optional = false python-versions = "*" +[package.dependencies] +importlib-resources = {version = ">=5.0", markers = "python_version < \"3.10\""} + [[package]] name = "flask" version = "2.0.2" @@ -317,6 +306,21 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "importlib-resources" +version = "5.10.2" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + [[package]] name = "itsdangerous" version = "2.1.2" @@ -377,7 +381,7 @@ zookeeper = ["kazoo (>=1.3.1)"] [[package]] name = "lxml" -version = "4.8.0" +version = "4.9.2" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false @@ -405,17 +409,6 @@ category = "main" optional = false python-versions = ">=3.7,<3.11" -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - [[package]] name = "parse" version = "1.19.0" @@ -434,7 +427,7 @@ python-versions = ">=3.6" [[package]] name = "prompt-toolkit" -version = "3.0.29" +version = "3.0.36" description = "Library for building powerful interactive command lines in Python" category = "main" optional = false @@ -462,17 +455,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "pyparsing" -version = "3.0.8" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["railroad-diagrams", "jinja2"] - [[package]] name = "pyppeteer" version = "0.0.14" @@ -487,16 +469,19 @@ websockets = "*" [[package]] name = "pyquery" -version = "1.4.3" +version = "2.0.0" description = "A jquery-like library for python" category = "main" optional = false python-versions = "*" [package.dependencies] -cssselect = ">0.7.9" +cssselect = ">=1.2.0" lxml = ">=2.1" +[package.extras] +test = ["requests", "webob", "webtest", "pytest", "pytest-cov"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -510,7 +495,7 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2022.1" +version = "2022.7" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -538,16 +523,14 @@ lxml = "*" [[package]] name = "redis" -version = "4.2.2" +version = "4.4.0" description = "Python client for Redis database and key-value store" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] async-timeout = ">=4.0.2" -deprecated = ">=1.2.3" -packaging = ">=20.4" [package.extras] hiredis = ["hiredis (>=1.0.0)"] @@ -602,6 +585,36 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "sentry-sdk" +version = "1.5.8" +description = "Python client for Sentry (https://sentry.io)" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +flask = ["flask (>=0.11)", "blinker (>=1.1)"] +httpx = ["httpx (>=0.16.0)"] +pure_eval = ["pure-eval", "executing", "asttokens"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["quart (>=0.16.1)", "blinker (>=1.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +tornado = ["tornado (>=5)"] + [[package]] name = "six" version = "1.16.0" @@ -620,7 +633,7 @@ python-versions = ">=3.6" [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -649,14 +662,11 @@ python-versions = ">=3.6" [[package]] name = "w3lib" -version = "1.22.0" +version = "2.1.1" description = "Library of web-related functions" category = "main" optional = false -python-versions = "*" - -[package.dependencies] -six = ">=1.4.1" +python-versions = ">=3.7" [[package]] name = "wcwidth" @@ -668,7 +678,7 @@ python-versions = "*" [[package]] name = "websockets" -version = "10.3" +version = "10.4" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" category = "main" optional = false @@ -676,579 +686,98 @@ python-versions = ">=3.7" [[package]] name = "werkzeug" -version = "2.1.2" +version = "2.2.2" description = "The comprehensive WSGI web application library." category = "main" optional = false python-versions = ">=3.7" +[package.dependencies] +MarkupSafe = ">=2.1.1" + [package.extras] watchdog = ["watchdog"] [[package]] -name = "wrapt" -version = "1.14.0" -description = "Module for decorators, wrappers and monkey patching." +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [metadata] lock-version = "1.1" -python-versions = ">=3.10, <3.11" -content-hash = "543ca98003a4d0f8f245c7e300624f0d1ea8e157fd924ea3b86619aeda203183" +python-versions = ">=3.8.10,<3.11" +content-hash = "efc561a94de9b16a5c1b9e8e217407cfda2fc8ce072d80a9f0f4f04588fcac28" [metadata.files] -amqp = [ - {file = "amqp-5.1.1-py3-none-any.whl", hash = "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359"}, - {file = "amqp-5.1.1.tar.gz", hash = "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2"}, -] -anytree = [ - {file = "anytree-2.8.0-py2.py3-none-any.whl", hash = "sha256:14c55ac77492b11532395049a03b773d14c7e30b22aa012e337b1e983de31521"}, - {file = "anytree-2.8.0.tar.gz", hash = "sha256:3f0f93f355a91bc3e6245319bf4c1d50e3416cc7a35cc1133c1ff38306bbccab"}, -] +amqp = [] +anytree = [] async-timeout = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, ] -beautifulsoup4 = [ - {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, - {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, -] -billiard = [ - {file = "billiard-3.6.4.0-py3-none-any.whl", hash = "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b"}, - {file = "billiard-3.6.4.0.tar.gz", hash = "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547"}, -] -boto3 = [ - {file = "boto3-1.19.0-py3-none-any.whl", hash = "sha256:d468b1f63f22ccd6b4bfbdebe6fd0c0b4620f38276af965ed139fe3eb85d16bb"}, - {file = "boto3-1.19.0.tar.gz", hash = "sha256:f93fed6153f7def66f1b17e6794c6ec3bec46229b213d3fa63f1eca126f5e992"}, -] -botocore = [ - {file = "botocore-1.22.12-py3-none-any.whl", hash = "sha256:1d1094fb53ebe4535d8840fbd7c14aadb65bde7ff03a65f9a4f1d76bd03e16ff"}, - {file = "botocore-1.22.12.tar.gz", hash = "sha256:fc59b55e8c5dde64b017b2f114c25f8cce397b667e812aea7eafb4b59b49d7cb"}, -] -bs4 = [ - {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"}, -] -celery = [ - {file = "celery-5.2.3-py3-none-any.whl", hash = "sha256:8aacd02fc23a02760686d63dde1eb0daa9f594e735e73ea8fb15c2ff15cb608c"}, - {file = "celery-5.2.3.tar.gz", hash = "sha256:e2cd41667ad97d4f6a2f4672d1c6a6ebada194c619253058b5f23704aaadaa82"}, -] -certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, - {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -click-didyoumean = [ - {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"}, - {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"}, -] -click-plugins = [ - {file = "click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b"}, - {file = "click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"}, -] -click-repl = [ - {file = "click-repl-0.2.0.tar.gz", hash = "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8"}, - {file = "click_repl-0.2.0-py3-none-any.whl", hash = "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -cssselect = [ - {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, - {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"}, -] +beautifulsoup4 = [] +billiard = [] +boto3 = [] +botocore = [] +bs4 = [] +celery = [] +certifi = [] +charset-normalizer = [] +click = [] +click-didyoumean = [] +click-plugins = [] +click-repl = [] +colorama = [] +cssselect = [] deep-parser = [] -deprecated = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, -] -fake-useragent = [ - {file = "fake-useragent-0.1.11.tar.gz", hash = "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"}, -] -flask = [ - {file = "Flask-2.0.2-py3-none-any.whl", hash = "sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a"}, - {file = "Flask-2.0.2.tar.gz", hash = "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2"}, -] -func-timeout = [ - {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"}, -] -idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, -] -itsdangerous = [ - {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, - {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, -] -jinja2 = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] -jmespath = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] -kombu = [ - {file = "kombu-5.2.4-py3-none-any.whl", hash = "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4"}, - {file = "kombu-5.2.4.tar.gz", hash = "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610"}, -] -lxml = [ - {file = "lxml-4.8.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:e1ab2fac607842ac36864e358c42feb0960ae62c34aa4caaf12ada0a1fb5d99b"}, - {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28d1af847786f68bec57961f31221125c29d6f52d9187c01cd34dc14e2b29430"}, - {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b92d40121dcbd74831b690a75533da703750f7041b4bf951befc657c37e5695a"}, - {file = "lxml-4.8.0-cp27-cp27m-win32.whl", hash = "sha256:e01f9531ba5420838c801c21c1b0f45dbc9607cb22ea2cf132844453bec863a5"}, - {file = "lxml-4.8.0-cp27-cp27m-win_amd64.whl", hash = "sha256:6259b511b0f2527e6d55ad87acc1c07b3cbffc3d5e050d7e7bcfa151b8202df9"}, - {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1010042bfcac2b2dc6098260a2ed022968dbdfaf285fc65a3acf8e4eb1ffd1bc"}, - {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fa56bb08b3dd8eac3a8c5b7d075c94e74f755fd9d8a04543ae8d37b1612dd170"}, - {file = "lxml-4.8.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:31ba2cbc64516dcdd6c24418daa7abff989ddf3ba6d3ea6f6ce6f2ed6e754ec9"}, - {file = "lxml-4.8.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:31499847fc5f73ee17dbe1b8e24c6dafc4e8d5b48803d17d22988976b0171f03"}, - {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5f7d7d9afc7b293147e2d506a4596641d60181a35279ef3aa5778d0d9d9123fe"}, - {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a3c5f1a719aa11866ffc530d54ad965063a8cbbecae6515acbd5f0fae8f48eaa"}, - {file = "lxml-4.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6268e27873a3d191849204d00d03f65c0e343b3bcb518a6eaae05677c95621d1"}, - {file = "lxml-4.8.0-cp310-cp310-win32.whl", hash = "sha256:330bff92c26d4aee79c5bc4d9967858bdbe73fdbdbacb5daf623a03a914fe05b"}, - {file = "lxml-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2582b238e1658c4061ebe1b4df53c435190d22457642377fd0cb30685cdfb76"}, - {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2bfc7e2a0601b475477c954bf167dee6d0f55cb167e3f3e7cefad906e7759f6"}, - {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a1547ff4b8a833511eeaceacbcd17b043214fcdb385148f9c1bc5556ca9623e2"}, - {file = "lxml-4.8.0-cp35-cp35m-win32.whl", hash = "sha256:a9f1c3489736ff8e1c7652e9dc39f80cff820f23624f23d9eab6e122ac99b150"}, - {file = "lxml-4.8.0-cp35-cp35m-win_amd64.whl", hash = "sha256:530f278849031b0eb12f46cca0e5db01cfe5177ab13bd6878c6e739319bae654"}, - {file = "lxml-4.8.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:078306d19a33920004addeb5f4630781aaeabb6a8d01398045fcde085091a169"}, - {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:86545e351e879d0b72b620db6a3b96346921fa87b3d366d6c074e5a9a0b8dadb"}, - {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24f5c5ae618395ed871b3d8ebfcbb36e3f1091fd847bf54c4de623f9107942f3"}, - {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:bbab6faf6568484707acc052f4dfc3802bdb0cafe079383fbaa23f1cdae9ecd4"}, - {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7993232bd4044392c47779a3c7e8889fea6883be46281d45a81451acfd704d7e"}, - {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d6483b1229470e1d8835e52e0ff3c6973b9b97b24cd1c116dca90b57a2cc613"}, - {file = "lxml-4.8.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ad4332a532e2d5acb231a2e5d33f943750091ee435daffca3fec0a53224e7e33"}, - {file = "lxml-4.8.0-cp36-cp36m-win32.whl", hash = "sha256:db3535733f59e5605a88a706824dfcb9bd06725e709ecb017e165fc1d6e7d429"}, - {file = "lxml-4.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5f148b0c6133fb928503cfcdfdba395010f997aa44bcf6474fcdd0c5398d9b63"}, - {file = "lxml-4.8.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:8a31f24e2a0b6317f33aafbb2f0895c0bce772980ae60c2c640d82caac49628a"}, - {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:719544565c2937c21a6f76d520e6e52b726d132815adb3447ccffbe9f44203c4"}, - {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:c0b88ed1ae66777a798dc54f627e32d3b81c8009967c63993c450ee4cbcbec15"}, - {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fa9b7c450be85bfc6cd39f6df8c5b8cbd76b5d6fc1f69efec80203f9894b885f"}, - {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e9f84ed9f4d50b74fbc77298ee5c870f67cb7e91dcdc1a6915cb1ff6a317476c"}, - {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1d650812b52d98679ed6c6b3b55cbb8fe5a5460a0aef29aeb08dc0b44577df85"}, - {file = "lxml-4.8.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:80bbaddf2baab7e6de4bc47405e34948e694a9efe0861c61cdc23aa774fcb141"}, - {file = "lxml-4.8.0-cp37-cp37m-win32.whl", hash = "sha256:6f7b82934c08e28a2d537d870293236b1000d94d0b4583825ab9649aef7ddf63"}, - {file = "lxml-4.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e1fd7d2fe11f1cb63d3336d147c852f6d07de0d0020d704c6031b46a30b02ca8"}, - {file = "lxml-4.8.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5045ee1ccd45a89c4daec1160217d363fcd23811e26734688007c26f28c9e9e7"}, - {file = "lxml-4.8.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0c1978ff1fd81ed9dcbba4f91cf09faf1f8082c9d72eb122e92294716c605428"}, - {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cbf2ff155b19dc4d4100f7442f6a697938bf4493f8d3b0c51d45568d5666b5"}, - {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ce13d6291a5f47c1c8dbd375baa78551053bc6b5e5c0e9bb8e39c0a8359fd52f"}, - {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11527dc23d5ef44d76fef11213215c34f36af1608074561fcc561d983aeb870"}, - {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:60d2f60bd5a2a979df28ab309352cdcf8181bda0cca4529769a945f09aba06f9"}, - {file = "lxml-4.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:62f93eac69ec0f4be98d1b96f4d6b964855b8255c345c17ff12c20b93f247b68"}, - {file = "lxml-4.8.0-cp38-cp38-win32.whl", hash = "sha256:20b8a746a026017acf07da39fdb10aa80ad9877046c9182442bf80c84a1c4696"}, - {file = "lxml-4.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:891dc8f522d7059ff0024cd3ae79fd224752676447f9c678f2a5c14b84d9a939"}, - {file = "lxml-4.8.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b6fc2e2fb6f532cf48b5fed57567ef286addcef38c28874458a41b7837a57807"}, - {file = "lxml-4.8.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:74eb65ec61e3c7c019d7169387d1b6ffcfea1b9ec5894d116a9a903636e4a0b1"}, - {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:627e79894770783c129cc5e89b947e52aa26e8e0557c7e205368a809da4b7939"}, - {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:545bd39c9481f2e3f2727c78c169425efbfb3fbba6e7db4f46a80ebb249819ca"}, - {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5a58d0b12f5053e270510bf12f753a76aaf3d74c453c00942ed7d2c804ca845c"}, - {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ec4b4e75fc68da9dc0ed73dcdb431c25c57775383fec325d23a770a64e7ebc87"}, - {file = "lxml-4.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5804e04feb4e61babf3911c2a974a5b86f66ee227cc5006230b00ac6d285b3a9"}, - {file = "lxml-4.8.0-cp39-cp39-win32.whl", hash = "sha256:aa0cf4922da7a3c905d000b35065df6184c0dc1d866dd3b86fd961905bbad2ea"}, - {file = "lxml-4.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:dd10383f1d6b7edf247d0960a3db274c07e96cf3a3fc7c41c8448f93eac3fb1c"}, - {file = "lxml-4.8.0-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:2403a6d6fb61c285969b71f4a3527873fe93fd0abe0832d858a17fe68c8fa507"}, - {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:986b7a96228c9b4942ec420eff37556c5777bfba6758edcb95421e4a614b57f9"}, - {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6fe4ef4402df0250b75ba876c3795510d782def5c1e63890bde02d622570d39e"}, - {file = "lxml-4.8.0-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:f10ce66fcdeb3543df51d423ede7e238be98412232fca5daec3e54bcd16b8da0"}, - {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:730766072fd5dcb219dd2b95c4c49752a54f00157f322bc6d71f7d2a31fecd79"}, - {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93"}, - {file = "lxml-4.8.0.tar.gz", hash = "sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23"}, -] -markupsafe = [ - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, - {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, -] -numpy = [ - {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13"}, - {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218"}, - {file = "numpy-1.21.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf"}, - {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1"}, - {file = "numpy-1.21.2-cp37-cp37m-win32.whl", hash = "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310"}, - {file = "numpy-1.21.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2"}, - {file = "numpy-1.21.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931"}, - {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f"}, - {file = "numpy-1.21.2-cp38-cp38-win32.whl", hash = "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557"}, - {file = "numpy-1.21.2-cp38-cp38-win_amd64.whl", hash = "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d"}, - {file = "numpy-1.21.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737"}, - {file = "numpy-1.21.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f"}, - {file = "numpy-1.21.2-cp39-cp39-win32.whl", hash = "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad"}, - {file = "numpy-1.21.2-cp39-cp39-win_amd64.whl", hash = "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724"}, - {file = "numpy-1.21.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7"}, - {file = "numpy-1.21.2.zip", hash = "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -parse = [ - {file = "parse-1.19.0.tar.gz", hash = "sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b"}, -] -pillow = [ - {file = "Pillow-8.3.2-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4"}, - {file = "Pillow-8.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c"}, - {file = "Pillow-8.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a"}, - {file = "Pillow-8.3.2-cp310-cp310-win32.whl", hash = "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228"}, - {file = "Pillow-8.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875"}, - {file = "Pillow-8.3.2-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7"}, - {file = "Pillow-8.3.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550"}, - {file = "Pillow-8.3.2-cp36-cp36m-win32.whl", hash = "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073"}, - {file = "Pillow-8.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196"}, - {file = "Pillow-8.3.2-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b"}, - {file = "Pillow-8.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da"}, - {file = "Pillow-8.3.2-cp37-cp37m-win32.whl", hash = "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9"}, - {file = "Pillow-8.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3"}, - {file = "Pillow-8.3.2-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616"}, - {file = "Pillow-8.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6"}, - {file = "Pillow-8.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b"}, - {file = "Pillow-8.3.2-cp38-cp38-win32.whl", hash = "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341"}, - {file = "Pillow-8.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb"}, - {file = "Pillow-8.3.2-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9"}, - {file = "Pillow-8.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b"}, - {file = "Pillow-8.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441"}, - {file = "Pillow-8.3.2-cp39-cp39-win32.whl", hash = "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09"}, - {file = "Pillow-8.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624"}, - {file = "Pillow-8.3.2-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1"}, - {file = "Pillow-8.3.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96"}, - {file = "Pillow-8.3.2.tar.gz", hash = "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c"}, -] -prompt-toolkit = [ - {file = "prompt_toolkit-3.0.29-py3-none-any.whl", hash = "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752"}, - {file = "prompt_toolkit-3.0.29.tar.gz", hash = "sha256:bd640f60e8cecd74f0dc249713d433ace2ddc62b65ee07f96d358e0b152b6ea7"}, -] -pyee = [ - {file = "pyee-9.0.4-py2.py3-none-any.whl", hash = "sha256:9f066570130c554e9cc12de5a9d86f57c7ee47fece163bbdaa3e9c933cfbdfa5"}, - {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, -] -pymupdf = [ - {file = "PyMuPDF-1.18.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4546e426e00e3ace148adaf95e0952061391ea04ec285c7a4592e11b9a372d0b"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0dac1bc4c474dce4d0569b02eb11126f15d9a12d4855fddf21471cff75db493e"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-win32.whl", hash = "sha256:8e7d1036a135ecbe30d185d968556367b018c9954043e17408f1798c1d0a7685"}, - {file = "PyMuPDF-1.18.17-cp310-cp310-win_amd64.whl", hash = "sha256:87f5af9570e6b12191b05e23dfd3e1fd8940f2c0a1ec0773c6496205fdf24f7c"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3c7e01284d9832c3ba76f6eb5c6825827f26749eb20cfe20f633bf980b3d3d85"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ed93f0964b49033073704e4bfb0908046c799731ee1e357f2b4ec934e551b515"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c76115e81a8a1596fe127b9b8b45ec846cb6bcfa891460a2efad4eb145573a4"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-win32.whl", hash = "sha256:615ed1e4d13d404e219c4caff0c8c7e5557a331ffaeb162e12016562ff160178"}, - {file = "PyMuPDF-1.18.17-cp36-cp36m-win_amd64.whl", hash = "sha256:3e805320eb1c6d4957d66b06baa4c2726fd0bb84acd5114be08a69c0a071abe6"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d0b1a211bea5f98556dec57172675958e28a90818d44dc6b095efb37034854b8"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57a8c34a236e7021f6defda501eb2afa310687fd45cfb650211d09e9ed4252e7"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecacfef4760c8b451284940d5dbd9aeeb36fd0c1e495545c4f487196047e167d"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-win32.whl", hash = "sha256:20591b790035f5d0cebfcb0d12f336e7d2bab1d708d20a20c8a75e8c6bb51481"}, - {file = "PyMuPDF-1.18.17-cp37-cp37m-win_amd64.whl", hash = "sha256:a89a528e3218026ffb8ccceffaa813b4a0bb2e4b8edb3a44371b0f697b18349c"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9212f95bb2978ddf3669dd8feef09ea47c13603b58efee6a305b6fe4a8a6a621"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3c605f2f2936067f9e2ce7d76127bb040c4528a347222ae23d7f440af8965afd"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a21735d8306d058adb62dcdd71d70f32c614f23729abc3971c1eb9099712069c"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-win32.whl", hash = "sha256:c385f38bc7d299917c3c64a096447d8fb573373705c120efec316fd109f53de8"}, - {file = "PyMuPDF-1.18.17-cp38-cp38-win_amd64.whl", hash = "sha256:9fca5a5666e3a09c9b573d2edd230a0e51c379d1b9e11f7121c622f5795fcce5"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e4581e2bbee1392cac203022e93de259221f9dd2ebfd1ca7e2ac1f00d454cff"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ee986ff5ed40b633b7e4bf321658ab31f1285b01628e2830c4bddf0003677f36"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ac77aac8db26dfb118dcc1b5f7857f7b669e9a415a76cc2fb77fd8429af2d4f"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-win32.whl", hash = "sha256:4379bf78ecf4ba17291ba01582ca02e5ddbbc96004b31e8be1cc1b582795b880"}, - {file = "PyMuPDF-1.18.17-cp39-cp39-win_amd64.whl", hash = "sha256:b7dbdfa078069413b3ef442f2b21770477bb7c1ce724b4b332198ca7bc3efffe"}, - {file = "PyMuPDF-1.18.17.tar.gz", hash = "sha256:fa39ee5e91eae77818e07b6bb7e0cb0b402ad88e39a74b08626ce1c2150c5414"}, -] -pyparsing = [ - {file = "pyparsing-3.0.8-py3-none-any.whl", hash = "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"}, - {file = "pyparsing-3.0.8.tar.gz", hash = "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954"}, -] -pyppeteer = [ - {file = "pyppeteer-0.0.14.tar.gz", hash = "sha256:4e0409fb30bb717296432d5548f6a3407d78d9efcf7a17f308fdb42d43607d9f"}, -] -pyquery = [ - {file = "pyquery-1.4.3-py3-none-any.whl", hash = "sha256:1fc33b7699455ed25c75282bc8f80ace1ac078b0dda5a933dacbd8b1c1f83963"}, - {file = "pyquery-1.4.3.tar.gz", hash = "sha256:a388eefb6bc4a55350de0316fbd97cda999ae669b6743ae5b99102ba54f5aa72"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pytz = [ - {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, - {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, -] -pyyaml = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, -] -readable-content = [ - {file = "readable-content-0.1.2.tar.gz", hash = "sha256:94e6c41c95db814473a190170c861d7a2a20ce04b245c2ecbd20da9eef1f8f93"}, -] -redis = [ - {file = "redis-4.2.2-py3-none-any.whl", hash = "sha256:4e95f4ec5f49e636efcf20061a5a9110c20852f607cfca6865c07aaa8a739ee2"}, - {file = "redis-4.2.2.tar.gz", hash = "sha256:0107dc8e98a4f1d1d4aa00100e044287f77121a1e6d2085545c4b7fa94a7a27f"}, -] -requests = [ - {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, - {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, -] -requests-html = [ - {file = "requests-html-0.10.0.tar.gz", hash = "sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947"}, - {file = "requests_html-0.10.0-py3-none-any.whl", hash = "sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2"}, -] -s3transfer = [ - {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"}, - {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"}, -] +fake-useragent = [] +flask = [] +func-timeout = [] +idna = [] +importlib-resources = [] +itsdangerous = [] +jinja2 = [] +jmespath = [] +kombu = [] +lxml = [] +markupsafe = [] +numpy = [] +parse = [] +pillow = [] +prompt-toolkit = [] +pyee = [] +pymupdf = [] +pyppeteer = [] +pyquery = [] +python-dateutil = [] +pytz = [] +pyyaml = [] +readable-content = [] +redis = [] +requests = [] +requests-html = [] +s3transfer = [] +sentry-sdk = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -soupsieve = [ - {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, - {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, -] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] -urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, -] -vine = [ - {file = "vine-5.0.0-py2.py3-none-any.whl", hash = "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30"}, - {file = "vine-5.0.0.tar.gz", hash = "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e"}, -] -w3lib = [ - {file = "w3lib-1.22.0-py2.py3-none-any.whl", hash = "sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53"}, - {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, -] +soupsieve = [] +typing-extensions = [] +urllib3 = [] +vine = [] +w3lib = [] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] -websockets = [ - {file = "websockets-10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978"}, - {file = "websockets-10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500"}, - {file = "websockets-10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8"}, - {file = "websockets-10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f"}, - {file = "websockets-10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47"}, - {file = "websockets-10.3-cp310-cp310-win32.whl", hash = "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae"}, - {file = "websockets-10.3-cp310-cp310-win_amd64.whl", hash = "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079"}, - {file = "websockets-10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79"}, - {file = "websockets-10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e"}, - {file = "websockets-10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6"}, - {file = "websockets-10.3-cp37-cp37m-win32.whl", hash = "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1"}, - {file = "websockets-10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4"}, - {file = "websockets-10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36"}, - {file = "websockets-10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69"}, - {file = "websockets-10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c"}, - {file = "websockets-10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442"}, - {file = "websockets-10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76"}, - {file = "websockets-10.3-cp38-cp38-win32.whl", hash = "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559"}, - {file = "websockets-10.3-cp38-cp38-win_amd64.whl", hash = "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d"}, - {file = "websockets-10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094"}, - {file = "websockets-10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667"}, - {file = "websockets-10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680"}, - {file = "websockets-10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3"}, - {file = "websockets-10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8"}, - {file = "websockets-10.3-cp39-cp39-win32.whl", hash = "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582"}, - {file = "websockets-10.3-cp39-cp39-win_amd64.whl", hash = "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02"}, - {file = "websockets-10.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4"}, - {file = "websockets-10.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755"}, - {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, - {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, -] -werkzeug = [ - {file = "Werkzeug-2.1.2-py3-none-any.whl", hash = "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255"}, - {file = "Werkzeug-2.1.2.tar.gz", hash = "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6"}, -] -wrapt = [ - {file = "wrapt-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:5a9a1889cc01ed2ed5f34574c90745fab1dd06ec2eee663e8ebeefe363e8efd7"}, - {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:9a3ff5fb015f6feb78340143584d9f8a0b91b6293d6b5cf4295b3e95d179b88c"}, - {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4b847029e2d5e11fd536c9ac3136ddc3f54bc9488a75ef7d040a3900406a91eb"}, - {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:9a5a544861b21e0e7575b6023adebe7a8c6321127bb1d238eb40d99803a0e8bd"}, - {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:88236b90dda77f0394f878324cfbae05ae6fde8a84d548cfe73a75278d760291"}, - {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f0408e2dbad9e82b4c960274214af533f856a199c9274bd4aff55d4634dedc33"}, - {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9d8c68c4145041b4eeae96239802cfdfd9ef927754a5be3f50505f09f309d8c6"}, - {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:22626dca56fd7f55a0733e604f1027277eb0f4f3d95ff28f15d27ac25a45f71b"}, - {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:65bf3eb34721bf18b5a021a1ad7aa05947a1767d1aa272b725728014475ea7d5"}, - {file = "wrapt-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09d16ae7a13cff43660155383a2372b4aa09109c7127aa3f24c3cf99b891c330"}, - {file = "wrapt-1.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:debaf04f813ada978d7d16c7dfa16f3c9c2ec9adf4656efdc4defdf841fc2f0c"}, - {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:748df39ed634851350efa87690c2237a678ed794fe9ede3f0d79f071ee042561"}, - {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1807054aa7b61ad8d8103b3b30c9764de2e9d0c0978e9d3fc337e4e74bf25faa"}, - {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763a73ab377390e2af26042f685a26787c402390f682443727b847e9496e4a2a"}, - {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8529b07b49b2d89d6917cfa157d3ea1dfb4d319d51e23030664a827fe5fd2131"}, - {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:68aeefac31c1f73949662ba8affaf9950b9938b712fb9d428fa2a07e40ee57f8"}, - {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59d7d92cee84a547d91267f0fea381c363121d70fe90b12cd88241bd9b0e1763"}, - {file = "wrapt-1.14.0-cp310-cp310-win32.whl", hash = "sha256:3a88254881e8a8c4784ecc9cb2249ff757fd94b911d5df9a5984961b96113fff"}, - {file = "wrapt-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a242871b3d8eecc56d350e5e03ea1854de47b17f040446da0e47dc3e0b9ad4d"}, - {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a65bffd24409454b889af33b6c49d0d9bcd1a219b972fba975ac935f17bdf627"}, - {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9d9fcd06c952efa4b6b95f3d788a819b7f33d11bea377be6b8980c95e7d10775"}, - {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:db6a0ddc1282ceb9032e41853e659c9b638789be38e5b8ad7498caac00231c23"}, - {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:14e7e2c5f5fca67e9a6d5f753d21f138398cad2b1159913ec9e9a67745f09ba3"}, - {file = "wrapt-1.14.0-cp35-cp35m-win32.whl", hash = "sha256:6d9810d4f697d58fd66039ab959e6d37e63ab377008ef1d63904df25956c7db0"}, - {file = "wrapt-1.14.0-cp35-cp35m-win_amd64.whl", hash = "sha256:d808a5a5411982a09fef6b49aac62986274ab050e9d3e9817ad65b2791ed1425"}, - {file = "wrapt-1.14.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b77159d9862374da213f741af0c361720200ab7ad21b9f12556e0eb95912cd48"}, - {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36a76a7527df8583112b24adc01748cd51a2d14e905b337a6fefa8b96fc708fb"}, - {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0057b5435a65b933cbf5d859cd4956624df37b8bf0917c71756e4b3d9958b9e"}, - {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0a4ca02752ced5f37498827e49c414d694ad7cf451ee850e3ff160f2bee9d3"}, - {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8c6be72eac3c14baa473620e04f74186c5d8f45d80f8f2b4eda6e1d18af808e8"}, - {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:21b1106bff6ece8cb203ef45b4f5778d7226c941c83aaaa1e1f0f4f32cc148cd"}, - {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:493da1f8b1bb8a623c16552fb4a1e164c0200447eb83d3f68b44315ead3f9036"}, - {file = "wrapt-1.14.0-cp36-cp36m-win32.whl", hash = "sha256:89ba3d548ee1e6291a20f3c7380c92f71e358ce8b9e48161401e087e0bc740f8"}, - {file = "wrapt-1.14.0-cp36-cp36m-win_amd64.whl", hash = "sha256:729d5e96566f44fccac6c4447ec2332636b4fe273f03da128fff8d5559782b06"}, - {file = "wrapt-1.14.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:891c353e95bb11abb548ca95c8b98050f3620a7378332eb90d6acdef35b401d4"}, - {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23f96134a3aa24cc50614920cc087e22f87439053d886e474638c68c8d15dc80"}, - {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6807bcee549a8cb2f38f73f469703a1d8d5d990815c3004f21ddb68a567385ce"}, - {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6915682f9a9bc4cf2908e83caf5895a685da1fbd20b6d485dafb8e218a338279"}, - {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f2f3bc7cd9c9fcd39143f11342eb5963317bd54ecc98e3650ca22704b69d9653"}, - {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3a71dbd792cc7a3d772ef8cd08d3048593f13d6f40a11f3427c000cf0a5b36a0"}, - {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5a0898a640559dec00f3614ffb11d97a2666ee9a2a6bad1259c9facd01a1d4d9"}, - {file = "wrapt-1.14.0-cp37-cp37m-win32.whl", hash = "sha256:167e4793dc987f77fd476862d32fa404d42b71f6a85d3b38cbce711dba5e6b68"}, - {file = "wrapt-1.14.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d066ffc5ed0be00cd0352c95800a519cf9e4b5dd34a028d301bdc7177c72daf3"}, - {file = "wrapt-1.14.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d9bdfa74d369256e4218000a629978590fd7cb6cf6893251dad13d051090436d"}, - {file = "wrapt-1.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2498762814dd7dd2a1d0248eda2afbc3dd9c11537bc8200a4b21789b6df6cd38"}, - {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f24ca7953f2643d59a9c87d6e272d8adddd4a53bb62b9208f36db408d7aafc7"}, - {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b835b86bd5a1bdbe257d610eecab07bf685b1af2a7563093e0e69180c1d4af1"}, - {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b21650fa6907e523869e0396c5bd591cc326e5c1dd594dcdccac089561cacfb8"}, - {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:354d9fc6b1e44750e2a67b4b108841f5f5ea08853453ecbf44c81fdc2e0d50bd"}, - {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f83e9c21cd5275991076b2ba1cd35418af3504667affb4745b48937e214bafe"}, - {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:61e1a064906ccba038aa3c4a5a82f6199749efbbb3cef0804ae5c37f550eded0"}, - {file = "wrapt-1.14.0-cp38-cp38-win32.whl", hash = "sha256:28c659878f684365d53cf59dc9a1929ea2eecd7ac65da762be8b1ba193f7e84f"}, - {file = "wrapt-1.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:b0ed6ad6c9640671689c2dbe6244680fe8b897c08fd1fab2228429b66c518e5e"}, - {file = "wrapt-1.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3f7e671fb19734c872566e57ce7fc235fa953d7c181bb4ef138e17d607dc8a1"}, - {file = "wrapt-1.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87fa943e8bbe40c8c1ba4086971a6fefbf75e9991217c55ed1bcb2f1985bd3d4"}, - {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4775a574e9d84e0212f5b18886cace049a42e13e12009bb0491562a48bb2b758"}, - {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d57677238a0c5411c76097b8b93bdebb02eb845814c90f0b01727527a179e4d"}, - {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00108411e0f34c52ce16f81f1d308a571df7784932cc7491d1e94be2ee93374b"}, - {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d332eecf307fca852d02b63f35a7872de32d5ba8b4ec32da82f45df986b39ff6"}, - {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:01f799def9b96a8ec1ef6b9c1bbaf2bbc859b87545efbecc4a78faea13d0e3a0"}, - {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47045ed35481e857918ae78b54891fac0c1d197f22c95778e66302668309336c"}, - {file = "wrapt-1.14.0-cp39-cp39-win32.whl", hash = "sha256:2eca15d6b947cfff51ed76b2d60fd172c6ecd418ddab1c5126032d27f74bc350"}, - {file = "wrapt-1.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:bb36fbb48b22985d13a6b496ea5fb9bb2a076fea943831643836c9f6febbcfdc"}, - {file = "wrapt-1.14.0.tar.gz", hash = "sha256:8323a43bd9c91f62bb7d4be74cc9ff10090e7ef820e27bfe8815c57e68261311"}, -] +websockets = [] +werkzeug = [] +zipp = [] diff --git a/mockserver/pyproject.toml b/mockserver/pyproject.toml index b48b5030..9d1b92f5 100644 --- a/mockserver/pyproject.toml +++ b/mockserver/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["deep-dev "] [tool.poetry.dependencies] -python = ">=3.10, <3.11" +python = ">=3.8.10,<3.11" Flask = "==2.0.2" anytree = "==2.8.0" certifi = "==2021.5.30" @@ -19,7 +19,8 @@ urllib3 = "==1.26.6" Pillow = "==8.3.2" boto3 = "==1.19.0" celery = { version = "==5.2.3", extras = ["redis"] } -deep-parser = { git = "https://github.com/the-deep/deepex.git", branch = "master" } +deep-parser = { git = "https://github.com/the-deep/deepex.git", rev="360766f8d51dc26e4b507e449663dbca1be87e38", branch = "master" } +sentry-sdk = "==1.5.8" [tool.poetry.dev-dependencies] From b56e61b94b4b3d4ee7eb6410230be7a3580a84dd Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 4 Jan 2023 12:06:45 +0545 Subject: [PATCH 38/45] updated the methods for parser library; --- lambda_fns/extract_docs/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 81e10b3c..42fc3555 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -139,7 +139,7 @@ def get_extracted_content_links(file_path, file_name, mock): binary = base64.b64encode(f.read()) document = TextFromFile(stream=binary, ext="pdf") - entries, images = document.serial_extract_text(output_format="list") + entries, images = document.extract_text(output_format="list") except Exception as e: logging.error(f"Extraction failed: {str(e)}") return None, None, -1, -1 @@ -208,7 +208,7 @@ def get_extracted_content_links(file_path, file_name, mock): def get_extracted_text_web_links(link, file_name, mock=False): try: web_text = TextFromWeb(url=link) - entries = web_text.extract_text(output_format="list") + entries = web_text.extract_text(output_format="list", url=link) except Exception as e: logging.error(f"Extraction from website failed {e}") return None, None, -1, -1 From 35af4fa880416e7ce79a08b31e143f7e586c2bc7 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 5 Jan 2023 10:04:38 +0545 Subject: [PATCH 39/45] ecs vcpu and memory reduced; --- staging.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staging.tfvars b/staging.tfvars index 6feaa47f..9e8c1859 100644 --- a/staging.tfvars +++ b/staging.tfvars @@ -33,7 +33,7 @@ cidr_block = "172.16.0.0/16" ecs_task_execution_role = "ECSTaskExecutionRole" # ECS -fargate_cpu = "1024" -fargate_memory = "2048" +fargate_cpu = "256" +fargate_memory = "512" app_count = 1 app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser-staging" From f07893a694533a28f8279b90c6018912fb36906f Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Thu, 5 Jan 2023 10:43:29 +0545 Subject: [PATCH 40/45] ecs vcpu and memory updated in prod; --- prod.tfvars | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prod.tfvars b/prod.tfvars index 324d4d2d..f9cbc4ee 100644 --- a/prod.tfvars +++ b/prod.tfvars @@ -33,7 +33,7 @@ cidr_block = "172.18.0.0/16" ecs_task_execution_role = "ECSTaskExecutionRole" # ECS -fargate_cpu = "2048" -fargate_memory = "4096" +fargate_cpu = "512" +fargate_memory = "1024" app_count = 1 app_image = "961104659532.dkr.ecr.us-east-1.amazonaws.com/deepex-parser-prod" \ No newline at end of file From 7e8ae4e34933a5a1f0e2cecf02f735a4895e888b Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 20 Jan 2023 14:05:41 +0545 Subject: [PATCH 41/45] refactor and removed few non-required functions; --- lambda_fns/extract_docs/app.py | 162 ++------------------------------- 1 file changed, 9 insertions(+), 153 deletions(-) diff --git a/lambda_fns/extract_docs/app.py b/lambda_fns/extract_docs/app.py index 42fc3555..b48789ff 100644 --- a/lambda_fns/extract_docs/app.py +++ b/lambda_fns/extract_docs/app.py @@ -1,20 +1,13 @@ import requests +import tempfile import uuid -import json import base64 import logging import re import pathlib -from datetime import datetime -from enum import Enum -import boto3 -from botocore.exceptions import ClientError -import tempfile - import sentry_sdk - -from deep_parser import TextFromFile -from deep_parser import TextFromWeb +from enum import Enum +from datetime import datetime try: from content_types import ExtractContentType, UrlTypes @@ -34,6 +27,7 @@ ECS_CONTAINER_NAME ) except ImportError: + from deep_parser import TextFromFile, TextFromWeb from .content_types import ExtractContentType, UrlTypes from .config import ( EXTRACTED_FILE_NAME, @@ -53,10 +47,6 @@ logging.getLogger().setLevel(logging.INFO) -s3_client = boto3.client('s3', region_name=AWS_REGION) -sqs_client = boto3.client('sqs', region_name=AWS_REGION) -lambda_client = boto3.client('lambda', region_name=AWS_REGION) - extract_content_type = ExtractContentType() sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT, attach_stacktrace=True, traces_sample_rate=1.0) @@ -84,47 +74,6 @@ def extract_path(filepath): return None, None, None -def store_text_s3(data, bucket_name, key): - s3_client.put_object( - Body=data, - Bucket=bucket_name, - Key=key - ) - -def upload_file_to_s3( - url, - key, - bucketname -): - try: - session = boto3.Session() - s3_resource = session.resource("s3") - - r = requests.get(url, stream=True) - if r.status_code == 200: - bucket = s3_resource.Bucket(bucketname) - bucket.upload_fileobj(r.raw, key) - logging.info(f"The file is uploaded to the S3 bucket {bucketname}") - else: - return False - except Exception as e: - logging.error(str(e)) - return False - return True - -def download_file(filename_s3, bucketname, filename_local): - try: - s3_client.download_file( - bucketname, - filename_s3, - filename_local - ) - logging.info("The file is downloaded in the lambda /tmp directory.") - except ClientError as e: - return False - return True - - def get_words_count(text): if text: w = re.sub(r'[^\w\s]', '', text) @@ -177,32 +126,6 @@ def get_extracted_content_links(file_path, file_name, mock): # with open(full_path, 'rb') as data: # s3_images_path.append(f"{DOMAIN_NAME}{str(dir_images_path)}/{f}") return s3_file_path, s3_images_path, total_pages, total_words_count - else: - store_text_s3( - extracted_text, - DEST_BUCKET_NAME, - f"{str(s3_path_prefix)}/{file_name}" - ) - - local_temp_directory = pathlib.Path('/tmp', file_name) - local_temp_directory.mkdir(parents=True) if not local_temp_directory.exists() else None - # Note: commented for now - # images.save_images(directory_path=local_temp_directory) - - # for subdir, dirs, files in os.walk(local_temp_directory): - # for f in files: - # full_path = os.path.join(subdir, f) - # with open(full_path, 'rb') as data: - # store_text_s3( - # data, - # DEST_BUCKET_NAME, - # f"{str(s3_path_prefix)}/images/{f}" - # ) - - s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" - s3_images_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/images" - - return s3_file_path, s3_images_path, total_pages, total_words_count def get_extracted_text_web_links(link, file_name, mock=False): @@ -233,13 +156,6 @@ def get_extracted_text_web_links(link, file_name, mock=False): with open(dir_path / file_name, 'w') as f: f.write(extracted_text) s3_file_path = f'{DOMAIN_NAME}{str(dir_path)}/{file_name}' - else: - store_text_s3( - extracted_text, - DEST_BUCKET_NAME, - f"{str(s3_path_prefix)}/{file_name}" - ) - s3_file_path = f"s3://{DEST_BUCKET_NAME}/{str(s3_path_prefix)}/{file_name}" return s3_file_path, None, total_pages, total_words_count # No images extraction (lib doesn't support?) @@ -248,24 +164,7 @@ def handle_urls(url, mock=False): file_name = EXTRACTED_FILE_NAME - if url.startswith("s3"): - bucket_name, file_path, file_name = extract_path(url) - local_file_path = f"/tmp/{file_name}" - s3_client.download_file( - bucket_name, - file_path, - local_file_path - ) - try: - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( - local_file_path, file_name, mock - ) - extraction_status = ExtractionStatus.SUCCESS.value - except Exception: - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value - - elif content_type == UrlTypes.PDF.value: # assume it is http/https pdf weblink + if content_type == UrlTypes.PDF.value: # assume it is http/https pdf weblink s3_file_path = None s3_images_path = None @@ -301,53 +200,9 @@ def handle_urls(url, mock=False): elif content_type == UrlTypes.DOCX.value or content_type == UrlTypes.MSWORD.value or \ content_type == UrlTypes.XLSX.value or content_type == UrlTypes.XLS.value or \ content_type == UrlTypes.PPTX.value or content_type == UrlTypes.PPT.value: - - ext_type = content_type - tmp_filename = f"{uuid.uuid4().hex}.{ext_type}" - flag = False - if upload_file_to_s3(url, key=tmp_filename, bucketname=DOCS_CONVERSION_BUCKET_NAME): - payload = json.dumps({ - "file": tmp_filename, - "bucket": DOCS_CONVERSION_BUCKET_NAME, - "ext": ext_type, - "fromS3": 1 - }) - - docs_conversion_lambda_response = lambda_client.invoke( - FunctionName=DOCS_CONVERT_LAMBDA_FN_NAME, - InvocationType="RequestResponse", - Payload=payload - ) - docs_conversion_lambda_response_json = json.loads( - docs_conversion_lambda_response["Payload"].read().decode("utf-8") - ) - - if "statusCode" in docs_conversion_lambda_response_json and \ - docs_conversion_lambda_response_json["statusCode"] == 200: - bucket_name = docs_conversion_lambda_response_json["bucket"] - file_path = docs_conversion_lambda_response_json["file"] - filename = file_path.split("/")[-1] - - if download_file(file_path, bucket_name, f"/tmp/{filename}"): - s3_file_path, s3_images_path, total_pages, total_words_count = get_extracted_content_links( - f"/tmp/{filename}", file_name, mock - ) - if s3_file_path: - extraction_status = ExtractionStatus.SUCCESS.value - else: - extraction_status = ExtractionStatus.FAILED.value - else: - flag = True - else: - logging.error(f"Error occurred during file conversion. {docs_conversion_lambda_response_json['error']}") - flag = True - else: - logging.warn("Could not upload the file to s3.") - flag = True - - if flag: - s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 - extraction_status = ExtractionStatus.FAILED.value + logging.warn("Text extraction from docx, xlsx, pptx, doc, xls, ppt is not available in local setup.") + s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 + extraction_status = ExtractionStatus.FAILED.value elif content_type == UrlTypes.IMG.value: logging.warn("Text extraction from Images is not available.") s3_file_path, s3_images_path, total_pages, total_words_count = None, None, -1, -1 @@ -367,6 +222,7 @@ def run_fargate_task( url, callback_url ): + import boto3 ecs_client = boto3.client('ecs', region_name=AWS_REGION) response = ecs_client.run_task( From 6852eca6945263d6adc8efcd318b322df8e9cf9d Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 20 Jan 2023 16:29:21 +0545 Subject: [PATCH 42/45] dockerfile updated; --- lambda_fns/extract_docs/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lambda_fns/extract_docs/Dockerfile b/lambda_fns/extract_docs/Dockerfile index ad197a9a..907597fd 100644 --- a/lambda_fns/extract_docs/Dockerfile +++ b/lambda_fns/extract_docs/Dockerfile @@ -26,3 +26,5 @@ RUN apt-get update -y \ && rm -rf /var/lib/apt/lists/* COPY deepex_ecs/app.py content_types.py wget.py s3_handler.py config.py /code/ + +RUN download-chromium \ No newline at end of file From f61261459fd5a14b168d04bb80c679daa5360a9e Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 20 Jan 2023 16:30:41 +0545 Subject: [PATCH 43/45] updated the poetry requirements; --- lambda_fns/extract_docs/poetry.lock | 126 ++++++++++++++++++----- lambda_fns/extract_docs/pyproject.toml | 4 +- lambda_fns/extract_docs/requirements.txt | 12 --- 3 files changed, 105 insertions(+), 37 deletions(-) diff --git a/lambda_fns/extract_docs/poetry.lock b/lambda_fns/extract_docs/poetry.lock index f117bfc4..23b6f1ac 100644 --- a/lambda_fns/extract_docs/poetry.lock +++ b/lambda_fns/extract_docs/poetry.lock @@ -13,6 +13,14 @@ six = ">=1.9.0" dev = ["check-manifest"] test = ["coverage"] +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "beautifulsoup4" version = "4.11.1" @@ -90,6 +98,14 @@ python-versions = ">=3.5.0" [package.extras] unicode_backport = ["unicodedata2"] +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + [[package]] name = "cssselect" version = "1.2.0" @@ -114,7 +130,6 @@ func-timeout = "*" lxml = "*" numpy = "*" PyMuPDF = "*" -pyppeteer = "0.0.14" pyyaml = "*" readable-content = "*" requests-html = "*" @@ -123,7 +138,7 @@ requests-html = "*" type = "git" url = "https://github.com/the-deep/deepex.git" reference = "master" -resolved_reference = "360766f8d51dc26e4b507e449663dbca1be87e38" +resolved_reference = "fd0b014cc5a08b50d448c5584220d312d430674b" [[package]] name = "fake-useragent" @@ -133,6 +148,9 @@ category = "main" optional = false python-versions = "*" +[package.dependencies] +importlib-resources = {version = ">=5.0", markers = "python_version < \"3.10\""} + [[package]] name = "func-timeout" version = "4.3.5" @@ -149,6 +167,37 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.10.2" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + [[package]] name = "jmespath" version = "0.10.0" @@ -159,17 +208,17 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "lxml" -version = "4.9.1" +version = "4.9.2" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" [package.extras] -source = ["Cython (>=0.29.7)"] -htmlsoup = ["beautifulsoup4"] -html5 = ["html5lib"] cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["beautifulsoup4"] +source = ["Cython (>=0.29.7)"] [[package]] name = "numpy" @@ -197,15 +246,12 @@ python-versions = ">=3.6" [[package]] name = "pyee" -version = "9.0.4" +version = "8.2.2" description = "A port of node.js's EventEmitter to python." category = "main" optional = false python-versions = "*" -[package.dependencies] -typing-extensions = "*" - [[package]] name = "pymupdf" version = "1.18.17" @@ -216,28 +262,36 @@ python-versions = "*" [[package]] name = "pyppeteer" -version = "0.0.14" +version = "1.0.2" description = "Headless chrome/chromium automation library (unofficial port of puppeteer)" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7,<4.0" [package.dependencies] -pyee = "*" -websockets = "*" +appdirs = ">=1.4.3,<2.0.0" +certifi = ">=2021" +importlib-metadata = ">=1.4" +pyee = ">=8.1.0,<9.0.0" +tqdm = ">=4.42.1,<5.0.0" +urllib3 = ">=1.25.8,<2.0.0" +websockets = ">=10.0,<11.0" [[package]] name = "pyquery" -version = "1.4.3" +version = "2.0.0" description = "A jquery-like library for python" category = "main" optional = false python-versions = "*" [package.dependencies] -cssselect = ">0.7.9" +cssselect = ">=1.2.0" lxml = ">=2.1" +[package.extras] +test = ["requests", "webob", "webtest", "pytest", "pytest-cov"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -365,12 +419,21 @@ optional = false python-versions = ">=3.6" [[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" category = "main" optional = false -python-versions = ">=3.7" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] [[package]] name = "urllib3" @@ -401,24 +464,40 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + [metadata] lock-version = "1.1" -python-versions = ">=3.10, <3.11" -content-hash = "9d20a92ad2e27e42e64b062363f9cee3b36553d422b8994d81b7b28456d30ad4" +python-versions = ">=3.8, <3.11" +content-hash = "213a5c955297e2e62e59de8a56eae953559f4022e2d397953f00605f0b550f64" [metadata.files] anytree = [] +appdirs = [] beautifulsoup4 = [] boto3 = [] botocore = [] bs4 = [] certifi = [] charset-normalizer = [] +colorama = [] cssselect = [] deep-parser = [] fake-useragent = [] func-timeout = [] idna = [] +importlib-metadata = [] +importlib-resources = [] jmespath = [] lxml = [] numpy = [] @@ -440,7 +519,8 @@ six = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] soupsieve = [] -typing-extensions = [] +tqdm = [] urllib3 = [] w3lib = [] websockets = [] +zipp = [] diff --git a/lambda_fns/extract_docs/pyproject.toml b/lambda_fns/extract_docs/pyproject.toml index 4a217930..4199e309 100644 --- a/lambda_fns/extract_docs/pyproject.toml +++ b/lambda_fns/extract_docs/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["deep-dev "] [tool.poetry.dependencies] -python = ">=3.10, <3.11" +python = ">=3.8, <3.11" anytree = "==2.8.0" certifi = "==2021.5.30" charset-normalizer = "==2.0.4" @@ -17,7 +17,7 @@ requests = "==2.26.0" urllib3 = "==1.26.6" Pillow = "==8.3.2" boto3 = "==1.19.0" -deep-parser = { git = "https://github.com/the-deep/deepex.git", rev="360766f8d51dc26e4b507e449663dbca1be87e38", branch = "master" } +deep-parser = { git = "https://github.com/the-deep/deepex.git", rev="fd0b014cc5a08b50d448c5584220d312d430674b", branch = "master" } sentry-sdk = "==1.5.8" [tool.poetry.dev-dependencies] diff --git a/lambda_fns/extract_docs/requirements.txt b/lambda_fns/extract_docs/requirements.txt index b2708ffc..bfa11120 100644 --- a/lambda_fns/extract_docs/requirements.txt +++ b/lambda_fns/extract_docs/requirements.txt @@ -1,14 +1,2 @@ -anytree==2.8.0 -certifi==2021.5.30 -charset-normalizer==2.0.4 -idna==3.2 -numpy==1.21.2 -PyMuPDF==1.18.17 -PyYAML==5.4.1 requests==2.26.0 -six==1.16.0 -urllib3==1.26.6 -Pillow==8.3.2 -lxml==4.7.1 -deep-parser @ git+https://github.com/the-deep/deepex sentry-sdk==1.5.8 From 503dc5908d6487ab91aec0dd4c99431e4475d8a9 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Fri, 20 Jan 2023 16:31:16 +0545 Subject: [PATCH 44/45] updated the deep parser import; --- lambda_fns/extract_docs/deepex_ecs/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lambda_fns/extract_docs/deepex_ecs/app.py b/lambda_fns/extract_docs/deepex_ecs/app.py index 4d17c43a..afb53927 100644 --- a/lambda_fns/extract_docs/deepex_ecs/app.py +++ b/lambda_fns/extract_docs/deepex_ecs/app.py @@ -13,8 +13,7 @@ import sentry_sdk -from deep_parser import TextFromFile -from deep_parser import TextFromWeb +from deep_parser import TextFromFile, TextFromWeb try: from content_types import ExtractContentType, UrlTypes From f13dbf4914dadf9be3179252a863464d22daccd1 Mon Sep 17 00:00:00 2001 From: Ranjan Shrestha Date: Wed, 1 Nov 2023 09:54:30 +0545 Subject: [PATCH 45/45] updated the mock data related to preds, thresholds and selected tags; --- .../entry_predict_output_request/app.py | 459 ++++++++++-------- 1 file changed, 267 insertions(+), 192 deletions(-) diff --git a/lambda_fns/entry_predict_output_request/app.py b/lambda_fns/entry_predict_output_request/app.py index 7d13c0a9..42b5dc3f 100644 --- a/lambda_fns/entry_predict_output_request/app.py +++ b/lambda_fns/entry_predict_output_request/app.py @@ -20,203 +20,278 @@ mappings = get_all_mappings() categories = get_categories() -fake_data = { - 'primary_tags': { - 'sectors': [{ - 'Agriculture': 0.0013131533306455466, 'Education': 0.003010824160731357, - 'Food Security': 0.002566287973119567, 'Health': 2.677955230077108, - 'Livelihoods': 0.01722483797685096, 'Logistics': 0.003670748323202133, - 'Nutrition': 0.0041013412481668045, 'Protection': 0.028100471686700296, - 'Shelter': 0.0035644680749447573, 'WASH': 0.00885658950175879 - }], - 'subpillars_2d': [{ - 'At Risk->Number Of People At Risk': 0.00023104241032948875, - 'At Risk->Risk And Vulnerabilities': 0.006840221311261014, - 'Capacities & Response->International Response': 1.51390548675291, - 'Capacities & Response->Local Response': 0.0024619154282845557, - 'Capacities & Response->National Response': 0.19748103480006374, - 'Capacities & Response->Number Of People Reached/Response Gaps': 0.1326687938096572, - 'Humanitarian Conditions->Coping Mechanisms': 0.008473951473004289, - 'Humanitarian Conditions->Living Standards': 0.014394345796770519, - 'Humanitarian Conditions->Number Of People In Need': 0.002753498941479671, - 'Humanitarian Conditions->Physical And Mental Well Being': 0.02261752535293742, - 'Impact->Driver/Aggravating Factors': 0.0028069927602222093, - 'Impact->Impact On People': 0.0035386373796923594, - 'Impact->Impact On Systems, Services And Networks': 0.00474455507679118, - 'Impact->Number Of People Affected': 0.002435182492869596, - 'Priority Interventions->Expressed By Humanitarian Staff': 0.004984116689725355, - 'Priority Interventions->Expressed By Population': 0.0034277827944606543, - 'Priority Needs->Expressed By Humanitarian Staff': 0.0018360981872926156, - 'Priority Needs->Expressed By Population': 0.007651697378605604 - }], - 'subpillars_1d': [{ - 'Casualties->Dead': 0.0018779816205746359, - 'Casualties->Injured': 0.0009131004424908987, - 'Casualties->Missing': 0.0010629182305330266, - 'Context->Demography': 0.01951472795739466, - 'Context->Economy': 0.002760568168014288, - 'Context->Environment': 0.001610475469772753, - 'Context->Legal & Policy': 0.0028414463984870143, - 'Context->Politics': 0.0030019306965793175, - 'Context->Security & Stability': 0.0028423364380035887, - 'Context->Socio Cultural': 0.0024926103993921592, - 'Covid-19->Cases': 0.004972799797542393, - 'Covid-19->Contact Tracing': 0.00032880847216941987, - 'Covid-19->Deaths': 0.001167356436152333, - 'Covid-19->Hospitalization & Care': 0.0024493522487762497, - 'Covid-19->Restriction Measures': 0.005428578056718992, - 'Covid-19->Testing': 0.0018874364551392537, - 'Covid-19->Vaccination': 0.0011778841898949057, - 'Displacement->Intentions': 0.0004781786533146116, - 'Displacement->Local Integration': 0.006963967811316252, - 'Displacement->Pull Factors': 0.0003674881209635401, - 'Displacement->Push Factors': 0.0002446720680234501, - 'Displacement->Type/Numbers/Movements': 0.012378716890357043, - 'Humanitarian Access->Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps': 0.001155513591390658, - 'Humanitarian Access->Physical Constraints': 0.0014652756362920627, - 'Humanitarian Access->Population To Relief': 0.001666667767016119, - 'Humanitarian Access->Relief To Population': 0.011259256380385366, - 'Information And Communication->Communication Means And Preferences': 0.007581055563475405, - 'Information And Communication->Information Challenges And Barriers': 0.0003372832482758289, - 'Information And Communication->Knowledge And Info Gaps (Hum)': 0.0009009759297542688, - 'Information And Communication->Knowledge And Info Gaps (Pop)': 0.0007702910806983709, - 'Shock/Event->Hazard & Threats': 0.006979638609387304, - 'Shock/Event->Type And Characteristics': 0.00357941840775311, - 'Shock/Event->Underlying/Aggravating Factors': 0.006321697112391976 - }] - }, - 'secondary_tags': { - 'age': [{ - 'Adult (18 to 59 years old)': 0.04068703080217044, - 'Children/Youth (5 to 17 years old)': 0.024587836709212173, - 'Infants/Toddlers (<5 years old)': 0.04259871318936348, - 'Older Persons (60+ years old)': 0.006414494919972342 - }], - 'gender': [{ - 'Female': 1.403369450233352, - 'Male': 0.007781315997073596 - }], - 'affected_groups': [{ - 'Asylum Seekers': 0.0002758237987769487, 'Host': 0.00997524285181002, - 'IDP': 0.004761773787105261, 'Migrants': 0.000846206055333217, - 'Refugees': 0.0007048035968182376, 'Returnees': 0.007033202674169585 - }], - 'specific_needs_groups': [{ - 'Child Head of Household': 0.0002081420534523204, - 'Chronically Ill': 0.0029977605726312978, - 'Elderly Head of Household': 0.0029921820636705627, - 'Female Head of Household': 0.002415977602746959, - 'GBV survivors': 0.020530499899998687, - 'Indigenous people': 0.0028101496774559985, - 'LGBTQI+': 0.00022843408415366598, - 'Minorities': 0.009432899118480036, - 'Persons with Disability': 0.000918924031014155, - 'Pregnant or Lactating Women': 2.0397998848739936, - 'Single Women (including Widows)': 0.007506779511459172, - 'Unaccompanied or Separated Children': 0.00019092757914525768 - }] - } -} +fake_data = [ + { + "primary_tags":{ + "sectors":{ + "Agriculture":0.0015192615267421517, + "Cross":2.266681720228756, + "Education":0.02295325743034482, + "Food Security":0.061531245176281245, + "Health":0.140668327609698, + "Livelihoods":0.040182591016803465, + "Logistics":0.005464191199280322, + "Nutrition":9.93173974469149e-05, + "Protection":5.455834865570068, + "Shelter":0.01595675065699551, + "WASH":0.0018369699578865296 + }, + "subpillars_2d":{ + "At Risk->Number Of People At Risk":7.958003038766037e-05, + "At Risk->Risk And Vulnerabilities":0.14909714121710171, + "Capacities & Response->International Response":0.0005300973977060302, + "Capacities & Response->Local Response":3.6785016277463e-05, + "Capacities & Response->National Response":0.002878893385915195, + "Capacities & Response->Number Of People Reached/Response Gaps":0.0015827980435763798, + "Humanitarian Conditions->Coping Mechanisms":0.006730915305929052, + "Humanitarian Conditions->Living Standards":0.13377755307234251, + "Humanitarian Conditions->Number Of People In Need":0.00030424525903072203, + "Humanitarian Conditions->Physical And Mental Well Being":3.557107225060463, + "Impact->Driver/Aggravating Factors":0.30262467761834466, + "Impact->Impact On People":0.1379885245114565, + "Impact->Impact On Systems, Services And Networks":0.05514782969839871, + "Impact->Number Of People Affected":0.006860913126729429, + "Priority Interventions->Expressed By Humanitarian Staff":7.764241470593131e-05, + "Priority Interventions->Expressed By Population":1.9618319887134326e-05, + "Priority Needs->Expressed By Humanitarian Staff":1.2330993318495788e-05, + "Priority Needs->Expressed By Population":0.0005511248533506519 + }, + "subpillars_1d":{ + "Casualties->Dead":0.5277328766309298, + "Casualties->Injured":0.5389928352087736, + "Casualties->Missing":0.0024325139949926073, + "Context->Demography":0.03165506558226687, + "Context->Economy":0.010170067738120755, + "Context->Environment":0.010367161731290465, + "Context->Legal & Policy":0.001767287377585122, + "Context->Politics":0.012721888593990694, + "Context->Security & Stability":5.508916079998016, + "Context->Socio Cultural":0.003907638601958752, + "Covid-19->Cases":3.329320053227194e-05, + "Covid-19->Contact Tracing":1.0524275430932241e-07, + "Covid-19->Deaths":5.623477904248189e-05, + "Covid-19->Hospitalization & Care":9.923554292375533e-06, + "Covid-19->Restriction Measures":0.0002665956107312408, + "Covid-19->Testing":7.267150184845613e-05, + "Covid-19->Vaccination":7.220470042039569e-05, + "Displacement->Intentions":2.8656749388270937e-06, + "Displacement->Local Integration":7.402830026246823e-05, + "Displacement->Pull Factors":8.282385503359062e-06, + "Displacement->Push Factors":0.0006505369898290015, + "Displacement->Type/Numbers/Movements":0.011573675569267042, + "Humanitarian Access->Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps":0.0016117644716157683, + "Humanitarian Access->Physical Constraints":0.001055635675584199, + "Humanitarian Access->Population To Relief":3.701728701344109e-06, + "Humanitarian Access->Relief To Population":2.0293501537790608e-05, + "Information And Communication->Communication Means And Preferences":9.698896974441595e-06, + "Information And Communication->Information Challenges And Barriers":0.00012904413324577035, + "Information And Communication->Knowledge And Info Gaps (Hum)":0.002071594852688057, + "Information And Communication->Knowledge And Info Gaps (Pop)":0.008236645306977961, + "Shock/Event->Hazard & Threats":0.04653984991212686, + "Shock/Event->Type And Characteristics":0.005213278234891949, + "Shock/Event->Underlying/Aggravating Factors":0.08061402477324009 + } + }, + "secondary_tags":{ + "age":{ + "Adult (18 to 59 years old)":9.536670404486358e-05, + "Children/Youth (5 to 17 years old)":0.0010865677419739466, + "Infants/Toddlers (<5 years old)":5.143924949259278e-05, + "Older Persons (60+ years old)":0.00022308759071165696 + }, + "gender":{ + "Female":0.007455993650688065, + "Male":0.02926515298895538 + }, + "affected_groups":{ + "Asylum Seekers":2.1205758226835425e-07, + "Host":5.6676799431443214e-05, + "IDP":0.0016466826006459694, + "Migrants":4.5306819629331585e-06, + "Refugees":0.0003480630202202833, + "Returnees":0.0010307924821972847 + }, + "specific_needs_groups":{ + "Child Head of Household":1.5001008864209293e-07, + "Chronically Ill":8.45890200354107e-06, + "Elderly Head of Household":2.563743350947334e-06, + "Female Head of Household":2.0595144726886407e-06, + "GBV survivors":4.9045229388866574e-05, + "Indigenous people":7.791085408825893e-06, + "LGBTQI+":1.3531972748686322e-06, + "Minorities":0.0001522377880808728, + "Persons with Disability":1.2454439340302108e-06, + "Pregnant or Lactating Women":7.379314550719399e-08, + "Single Women (including Widows)":2.3683933629096527e-06, + "Unaccompanied or Separated Children":9.06970538178737e-07 + }, + "severity":{ + "Critical":0.45211867049888327, + "Major":0.028029342435977676, + "Minor Problem":0.0002331270297872834, + "No problem":0.006089172772287081, + "Of Concern":0.0023042623070068657 + } + } + } +] fake_data_thresholds = { - 'primary_tags': { - 'sectors': { - 'Agriculture': 0.41000000000000003, 'Education': 0.46, - 'Food Security': 0.48, 'Health': 0.36, 'Livelihoods': 0.38, - 'Logistics': 0.5, 'Nutrition': 0.49, 'Protection': 0.58, - 'Shelter': 0.42, 'WASH': 0.53 - }, - 'subpillars_2d': { - 'At Risk->Number Of People At Risk': 0.12, - 'At Risk->Risk And Vulnerabilities': 0.41000000000000003, - 'Capacities & Response->International Response': 0.62, - 'Capacities & Response->Local Response': 0.1, - 'Capacities & Response->National Response': 0.43, - 'Capacities & Response->Number Of People Reached/Response Gaps': 0.49, - 'Humanitarian Conditions->Coping Mechanisms': 0.36, - 'Humanitarian Conditions->Living Standards': 0.45, - 'Humanitarian Conditions->Number Of People In Need': 0.31, - 'Humanitarian Conditions->Physical And Mental Well Being': 0.41000000000000003, - 'Impact->Driver/Aggravating Factors': 0.38, 'Impact->Impact On People': 0.33, - 'Impact->Impact On Systems, Services And Networks': 0.45, - 'Impact->Number Of People Affected': 0.24, - 'Priority Interventions->Expressed By Humanitarian Staff': 0.55, - 'Priority Interventions->Expressed By Population': 0.15, - 'Priority Needs->Expressed By Humanitarian Staff': 0.3, - 'Priority Needs->Expressed By Population': 0.25 - }, - 'subpillars_1d': { - 'Casualties->Dead': 0.28, - 'Casualties->Injured': 0.13, - 'Casualties->Missing': 0.13, 'Context->Demography': 0.49, - 'Context->Economy': 0.41000000000000003, 'Context->Environment': 0.38, - 'Context->Legal & Policy': 0.31, 'Context->Politics': 0.3, - 'Context->Security & Stability': 0.44, 'Context->Socio Cultural': 0.17, - 'Covid-19->Cases': 0.8, 'Covid-19->Contact Tracing': 0.39, - 'Covid-19->Deaths': 0.81, 'Covid-19->Hospitalization & Care': 0.41000000000000003, - 'Covid-19->Restriction Measures': 0.46, 'Covid-19->Testing': 0.79, - 'Covid-19->Vaccination': 0.54, 'Displacement->Intentions': 0.38, - 'Displacement->Local Integration': 0.25, 'Displacement->Pull Factors': 0.29, - 'Displacement->Push Factors': 0.37, 'Displacement->Type/Numbers/Movements': 0.38, - 'Humanitarian Access->Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps': 0.47000000000000003, - 'Humanitarian Access->Physical Constraints': 0.48, - 'Humanitarian Access->Population To Relief': 0.19, - 'Humanitarian Access->Relief To Population': 0.29, - 'Information And Communication->Communication Means And Preferences': 0.21, - 'Information And Communication->Information Challenges And Barriers': 0.15, - 'Information And Communication->Knowledge And Info Gaps (Hum)': 0.18, - 'Information And Communication->Knowledge And Info Gaps (Pop)': 0.18, - 'Shock/Event->Hazard & Threats': 0.23, 'Shock/Event->Type And Characteristics': 0.2, - 'Shock/Event->Underlying/Aggravating Factors': 0.17 - } - }, - 'secondary_tags': { - 'age': { - 'Adult (18 to 59 years old)': 0.48, 'Children/Youth (5 to 17 years old)': 0.44, - 'Infants/Toddlers (<5 years old)': 0.4, 'Older Persons (60+ years old)': 0.61 - }, - 'gender': { - 'Female': 0.71, 'Male': 0.44 - }, - 'affected_groups': { - 'Asylum Seekers': 0.73, 'Host': 0.55, 'IDP': 0.67, 'Migrants': 0.75, - 'Refugees': 0.64, 'Returnees': 0.53 - }, - 'specific_needs_groups': { - 'Child Head of Household': 0.25, 'Chronically Ill': 0.58, 'Elderly Head of Household': 0.14, - 'Female Head of Household': 0.48, 'GBV survivors': 0.78, 'Indigenous people': 0.13, - 'LGBTQI+': 0.41000000000000003, 'Minorities': 0.59, 'Persons with Disability': 0.56, - 'Pregnant or Lactating Women': 0.49, 'Single Women (including Widows)': 0.2, - 'Unaccompanied or Separated Children': 0.6 - } - } + "primary_tags":{ + "sectors":{ + "Agriculture":0.14, + "Cross":0.17, + "Education":0.1, + "Food Security":0.14, + "Health":0.18, + "Livelihoods":0.14, + "Logistics":0.1, + "Nutrition":0.12, + "Protection":0.15, + "Shelter":0.18, + "WASH":0.14 + }, + "subpillars_2d":{ + "At Risk->Number Of People At Risk":0.01, + "At Risk->Risk And Vulnerabilities":0.11, + "Capacities & Response->International Response":0.38, + "Capacities & Response->Local Response":0.01, + "Capacities & Response->National Response":0.17, + "Capacities & Response->Number Of People Reached/Response Gaps":0.15, + "Humanitarian Conditions->Coping Mechanisms":0.09, + "Humanitarian Conditions->Living Standards":0.13, + "Humanitarian Conditions->Number Of People In Need":0.07, + "Humanitarian Conditions->Physical And Mental Well Being":0.16, + "Impact->Driver/Aggravating Factors":0.15, + "Impact->Impact On People":0.2, + "Impact->Impact On Systems, Services And Networks":0.16, + "Impact->Number Of People Affected":0.05, + "Priority Interventions->Expressed By Humanitarian Staff":0.45, + "Priority Interventions->Expressed By Population":0.06, + "Priority Needs->Expressed By Humanitarian Staff":0.28, + "Priority Needs->Expressed By Population":0.13 + }, + "subpillars_1d":{ + "Casualties->Dead":0.13, + "Casualties->Injured":0.04, + "Casualties->Missing":0.09, + "Context->Demography":0.14, + "Context->Economy":0.24, + "Context->Environment":0.17, + "Context->Legal & Policy":0.47, + "Context->Politics":0.22, + "Context->Security & Stability":0.16, + "Context->Socio Cultural":0.15, + "Covid-19->Cases":0.72, + "Covid-19->Contact Tracing":0.55, + "Covid-19->Deaths":0.61, + "Covid-19->Hospitalization & Care":0.3, + "Covid-19->Restriction Measures":0.23, + "Covid-19->Testing":0.31, + "Covid-19->Vaccination":0.39, + "Displacement->Intentions":0.15, + "Displacement->Local Integration":0.13, + "Displacement->Pull Factors":0.09, + "Displacement->Push Factors":0.26, + "Displacement->Type/Numbers/Movements":0.31, + "Humanitarian Access->Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps":0.09, + "Humanitarian Access->Physical Constraints":0.16, + "Humanitarian Access->Population To Relief":0.16, + "Humanitarian Access->Relief To Population":0.22, + "Information And Communication->Communication Means And Preferences":0.21, + "Information And Communication->Information Challenges And Barriers":0.04, + "Information And Communication->Knowledge And Info Gaps (Hum)":0.07, + "Information And Communication->Knowledge And Info Gaps (Pop)":0.09, + "Shock/Event->Hazard & Threats":0.24, + "Shock/Event->Type And Characteristics":0.21, + "Shock/Event->Underlying/Aggravating Factors":0.05 + } + }, + "secondary_tags":{ + "age":{ + "Adult (18 to 59 years old)":0.06, + "Children/Youth (5 to 17 years old)":0.48, + "Infants/Toddlers (<5 years old)":0.34, + "Older Persons (60+ years old)":0.16 + }, + "gender":{ + "Female":0.45, + "Male":0.48 + }, + "affected_groups":{ + "Asylum Seekers":0.66, + "Host":0.3, + "IDP":0.36, + "Migrants":0.23, + "Refugees":0.58, + "Returnees":0.3 + }, + "specific_needs_groups":{ + "Child Head of Household":0.29, + "Chronically Ill":0.45, + "Elderly Head of Household":0.03, + "Female Head of Household":0.34, + "GBV survivors":0.37, + "Indigenous people":0.25, + "LGBTQI+":0.07, + "Minorities":0.11, + "Persons with Disability":0.43, + "Pregnant or Lactating Women":0.23, + "Single Women (including Widows)":0.06, + "Unaccompanied or Separated Children":0.36 + }, + "severity":{ + "Critical":0.27, + "Major":0.11, + "Minor Problem":0.05, + "No problem":0.24, + "Of Concern":0.12 + } + } } fake_selected_tags = { - 'sectors': [ - ['Health'] - ], - 'subpillars_2d': [ - ['Capacities & Response->International Response'] - ], - 'subpillars_1d': [ - [] - ], - 'gender': [ - ['Female'] - ], - 'age': [ - ['Adult (18 to 59 years old)'] - ], - 'severity': [ - [] - ], - 'specific_needs_groups': [ - ['Pregnant or Lactating Women'] - ], - 'affected_groups': [ - [] - ] + "sectors":[ + [ + "Cross", + "Protection" + ] + ], + "subpillars_2d":[ + [ + "Humanitarian Conditions->Physical And Mental Well Being" + ] + ], + "subpillars_1d":[ + [ + "Context->Security & Stability" + ] + ], + "age":[ + [ + + ] + ], + "gender":[ + [ + + ] + ], + "affected_groups":[ + [ + + ] + ], + "specific_needs_groups":[ + [ + + ] + ], + "severity":[ + [ + "Critical" + ] + ] }