From 478315e671992b5615f4dbb0d481f61251bda538 Mon Sep 17 00:00:00 2001 From: damian0604 Date: Wed, 13 Oct 2021 14:37:02 +0000 Subject: [PATCH 1/5] add upload function --- inca2amcat.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 inca2amcat.py diff --git a/inca2amcat.py b/inca2amcat.py new file mode 100755 index 0000000..4d03afa --- /dev/null +++ b/inca2amcat.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import json +from tqdm import tqdm +from amcat4apiclient.amcat4apiclient import AmcatClient + + +amcat = AmcatClient("https://tux01ascor.fmg.uva.nl/amcat4server", "admin", "whatever") +indices = amcat.list_indices() +for index in indices: + print(index) + + + +with open('TEST.json') as f: + data = [json.loads(line) for line in f] + + +def chunker(x, chunksize = 100): + for i in range(0, len(x), chunksize): + slice_item = slice(i, i + chunksize, 1) + yield x[slice_item] + + +def cleanart(art): + art['date'] = art.pop('publication_date','1900-01-01') + if 'text' not in art: + art['text']='' + return art + +# data2 = [cleanart(art) for art in data] + +for chunk in tqdm(chunker(data)): + cleanchunk = [cleanart(art) for art in chunk] + r = amcat.upload('incatransfer', cleanchunk) + print(r.status_code) From 391e55c231be8b5d65ed6fdb9ee67ce2380a47e3 Mon Sep 17 00:00:00 2001 From: damian0604 Date: Wed, 13 Oct 2021 15:20:04 +0000 Subject: [PATCH 2/5] upload functionality works; batchuploader as demo --- amcat4apiclient/amcat4apiclient.py | 11 ++++ demo-batchuploader.py | 80 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100755 demo-batchuploader.py diff --git a/amcat4apiclient/amcat4apiclient.py b/amcat4apiclient/amcat4apiclient.py index 3d49a66..25f32ed 100644 --- a/amcat4apiclient/amcat4apiclient.py +++ b/amcat4apiclient/amcat4apiclient.py @@ -51,3 +51,14 @@ def query(self, index: str, q: Optional[str]= None, *, params['scroll_id'] = d['meta']['scroll_id'] + def upload(self, index: str, documents: list): + """ + Upload a set of documents to the server + + :param index: The name of the index + :param documents: A list of dictionaries with at least the keys date, title, text + :return: response of the POST request to the server + """ + url = f"{self.host}/index/{index}/documents" + r = requests.post(url, auth=(self.username, self.password), json=documents) + return r diff --git a/demo-batchuploader.py b/demo-batchuploader.py new file mode 100755 index 0000000..db06316 --- /dev/null +++ b/demo-batchuploader.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import json +import gzip +import argparse +import os +from typing import List, Dict +from tqdm import tqdm +from glob import glob +from amcat4apiclient.amcat4apiclient import AmcatClient + + +''' + + + + + + +# + + +''' + + +def _chunker(x, chunksize = 100): + for i in range(0, len(x), chunksize): + slice_item = slice(i, i + chunksize, 1) + yield x[slice_item] + + +def _cleandoc(doc: dict): + ''' + Ensures that document conforms to AmCAT requirements + ''' + # rename 'publication_date' to 'date'; handle missing dates + doc['date'] = doc.pop('publication_date','1900-01-01') + # handle missing text + if 'text' not in doc: doc['text']='' + return doc + + + +def read_file(fn, jsonlines=True): + if not jsonlines: + raise NotImplementedError("Still need to import logic to support both JSON and JSON-lines") + + if fn[-3:].lower()=='.gz': + with gzip.open(fn, "rb") as f: + data = [json.loads(line) for line in f] + else: + with open(fn, "rb") as f: + data = [json.loads(line) for line in f] + return data + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__, + epilog = "Set the environment variables AMCATUSER and AMCATPASSWORD to use non-default credentials") + parser.add_argument('index', help="The name of the index ('project') to upload to") + parser.add_argument('url', default='http://127.0.0.1:5000', help='The address of the AmCAT server') + parser.add_argument('files', help='Glob pattern of json(.gz) files') + + args = parser.parse_args() + + user = os.environ.get("AMCATUSER","admin") + passwd = os.environ.get("AMCATPASSWORD","admin") + + + amcat = AmcatClient(args.url, user, passwd) + + + allfiles = glob(args.files) + + for fn in tqdm(allfiles): + print(f"Processing {fn}...") + data = read_file(fn) + + for chunk in tqdm(_chunker(data)): + cleanchunk = [_cleandoc(art) for art in chunk] + r = amcat.upload('incatransfer', cleanchunk) + print(r.status_code) From 799b17fd488aa067f78be4060910fde1a056a995 Mon Sep 17 00:00:00 2001 From: damian0604 Date: Wed, 13 Oct 2021 15:22:54 +0000 Subject: [PATCH 3/5] remove temp file --- inca2amcat.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100755 inca2amcat.py diff --git a/inca2amcat.py b/inca2amcat.py deleted file mode 100755 index 4d03afa..0000000 --- a/inca2amcat.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -import json -from tqdm import tqdm -from amcat4apiclient.amcat4apiclient import AmcatClient - - -amcat = AmcatClient("https://tux01ascor.fmg.uva.nl/amcat4server", "admin", "whatever") -indices = amcat.list_indices() -for index in indices: - print(index) - - - -with open('TEST.json') as f: - data = [json.loads(line) for line in f] - - -def chunker(x, chunksize = 100): - for i in range(0, len(x), chunksize): - slice_item = slice(i, i + chunksize, 1) - yield x[slice_item] - - -def cleanart(art): - art['date'] = art.pop('publication_date','1900-01-01') - if 'text' not in art: - art['text']='' - return art - -# data2 = [cleanart(art) for art in data] - -for chunk in tqdm(chunker(data)): - cleanchunk = [cleanart(art) for art in chunk] - r = amcat.upload('incatransfer', cleanchunk) - print(r.status_code) From 0fcf5cfc834b691eaaff19c70c9eda587ad9d527 Mon Sep 17 00:00:00 2001 From: damian0604 Date: Wed, 13 Oct 2021 15:25:51 +0000 Subject: [PATCH 4/5] fix docstring --- demo-batchuploader.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/demo-batchuploader.py b/demo-batchuploader.py index db06316..c434dad 100755 --- a/demo-batchuploader.py +++ b/demo-batchuploader.py @@ -1,4 +1,9 @@ #!/usr/bin/env python3 +''' +Uploads multiple (gzipped or not) JSON files to AmCAT +''' + + import json import gzip import argparse @@ -9,18 +14,6 @@ from amcat4apiclient.amcat4apiclient import AmcatClient -''' - - - - - - -# - - -''' - def _chunker(x, chunksize = 100): for i in range(0, len(x), chunksize): From 796af43004bf9d4ae03f29d2cdfa3657da6fdf8c Mon Sep 17 00:00:00 2001 From: damian0604 Date: Fri, 15 Oct 2021 15:52:00 +0000 Subject: [PATCH 5/5] generator to avoid OOM with large files; pylint --- demo-batchuploader.py | 49 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/demo-batchuploader.py b/demo-batchuploader.py index c434dad..c2a5d55 100755 --- a/demo-batchuploader.py +++ b/demo-batchuploader.py @@ -8,23 +8,27 @@ import gzip import argparse import os -from typing import List, Dict -from tqdm import tqdm from glob import glob +from tqdm import tqdm from amcat4apiclient.amcat4apiclient import AmcatClient -def _chunker(x, chunksize = 100): - for i in range(0, len(x), chunksize): - slice_item = slice(i, i + chunksize, 1) - yield x[slice_item] +def _chunker(iterable, chunksize=100): + '''Yield successive chunks from an iterable (e.g., list, generator)''' + chunk = [] + for item in iterable: + if len(chunk) >= chunksize: + yield chunk + chunk = [item] + else: + chunk.append(item) + if chunk: + yield chunk def _cleandoc(doc: dict): - ''' - Ensures that document conforms to AmCAT requirements - ''' + '''Ensure that document conforms to AmCAT requirements''' # rename 'publication_date' to 'date'; handle missing dates doc['date'] = doc.pop('publication_date','1900-01-01') # handle missing text @@ -32,41 +36,38 @@ def _cleandoc(doc: dict): return doc - def read_file(fn, jsonlines=True): if not jsonlines: raise NotImplementedError("Still need to import logic to support both JSON and JSON-lines") if fn[-3:].lower()=='.gz': with gzip.open(fn, "rb") as f: - data = [json.loads(line) for line in f] + for line in f: + yield json.loads(line) else: with open(fn, "rb") as f: - data = [json.loads(line) for line in f] - return data - + for line in f: + yield json.loads(line) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__, - epilog = "Set the environment variables AMCATUSER and AMCATPASSWORD to use non-default credentials") - parser.add_argument('index', help="The name of the index ('project') to upload to") - parser.add_argument('url', default='http://127.0.0.1:5000', help='The address of the AmCAT server') + epilog = "Set the environment variables AMCATUSER and "\ + "AMCATPASSWORD to use non-default credentials") + parser.add_argument('index', + help="The name of the index ('project') to upload to") + parser.add_argument('url', default='http://127.0.0.1:5000', + help='The address of the AmCAT server') parser.add_argument('files', help='Glob pattern of json(.gz) files') args = parser.parse_args() - user = os.environ.get("AMCATUSER","admin") passwd = os.environ.get("AMCATPASSWORD","admin") - - amcat = AmcatClient(args.url, user, passwd) - - allfiles = glob(args.files) - for fn in tqdm(allfiles): print(f"Processing {fn}...") data = read_file(fn) - for chunk in tqdm(_chunker(data)): cleanchunk = [_cleandoc(art) for art in chunk] r = amcat.upload('incatransfer', cleanchunk)