Merge pull request #19 from ai-forever/dev

New features

Author: boomb0om

.gitignore

@@ -3,6 +3,9 @@ data/
 tests/
 pipelines/
 
+# pycharm
+.idea/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

DPF/filesystems/filesystem.py

@@ -2,10 +2,33 @@
 import os
 import io
 import tarfile
-from typing import Union, List, Tuple, Iterable
+import datetime
+from typing import Union, List, Dict, Optional, Tuple, Iterable
 import pandas as pd
 
 
+class FileData:
+    """Class that represents a file with his metadata"""
+
+    def __init__(
+        self,
+        path: str,
+        type: str,
+        last_modified: Optional[datetime.datetime] = None,
+        file_size: Optional[int] = None
+    ):
+        self.path = path
+        self.name = os.path.basename(self.path.rstrip('/'))
+        assert type in {'directory', 'file'}, \
+            "param 'type' must be one of {'directory', 'file'}, got "+str(type)
+        self.type = type
+        self.last_modified = last_modified
+        self.file_size = file_size
+
+    def __repr__(self) -> str:
+        return f'File(path="{self.path}, size={self.file_size}, last_modified={self.last_modified}")'
+
+
 class FileSystem(ABC):
     """
     Abstract class for all filesystems
@@ -153,6 +176,22 @@ def listdir_with_ext(
             if f.endswith(ext)
         ]
 
+    @abstractmethod
+    def listdir_meta(self, folder_path: str) -> List[FileData]:
+        """
+        Returns the contents of folder with meta information (datetime created, etc)
+
+        Parameters
+        ----------
+        folder_path: str
+            Path to folder
+
+        Returns
+        -------
+        List[FileData]
+            List of FileData objects
+        """
+
     @abstractmethod
     def mkdir(self, folder_path: str) -> None:
         """

DPF/filesystems/localfilesystem.py

@@ -1,8 +1,9 @@
 import os
 import io
+from datetime import datetime
 from typing import Union, List, Optional, Tuple, Iterable
 
-from .filesystem import FileSystem
+from .filesystem import FileSystem, FileData
 
 
 class LocalFileSystem(FileSystem):
@@ -44,6 +45,18 @@ def listdir(
             files = [folder_path + f for f in files]
         return files
 
+    def listdir_meta(self, folder_path: str) -> List[FileData]:
+        folder_path = folder_path.rstrip("/") + "/"
+        results = []
+        for fd in os.scandir(folder_path):
+            path = fd.path
+            type_ = 'directory' if fd.is_dir() else 'file'
+            stats = fd.stat()
+            size = stats.st_size
+            last_modified = datetime.fromtimestamp(stats.st_mtime)
+            results.append(FileData(path, type_, last_modified, size))
+        return results
+
     def mkdir(self, folder_path: str) -> None:
         folder_path = folder_path.rstrip("/") + "/"
         os.makedirs(folder_path, exist_ok=True)

DPF/filesystems/s3filesystem.py

@@ -3,7 +3,7 @@
 from typing import Union, List, Optional, Tuple, Iterable
 import fsspec
 
-from .filesystem import FileSystem
+from .filesystem import FileSystem, FileData
 
 
 class S3FileSystem(FileSystem):
@@ -63,6 +63,24 @@ def listdir(
             files = ["s3://" + f for f in files]
         return files
 
+    def listdir_meta(self, folder_path: str) -> List[FileData]:
+        folder_path = folder_path.lstrip("s3://").rstrip("/") + "/"
+        s3 = fsspec.filesystem("s3", **self.storage_options)
+        files_data = s3.ls(folder_path, detail=True)
+
+        results = []
+        for file_data in files_data:
+            if file_data['Key'] == folder_path:
+                continue
+            path = "s3://"+file_data['Key']
+            filetype = file_data['type']
+            size = None
+            last_modified = file_data.get('LastModified', None)
+            if filetype == 'file':
+                size = file_data.get('Size', None)
+            results.append(FileData(path, filetype, last_modified, size))
+        return results
+
     def mkdir(self, folder_path: str) -> None:
         folder_path = folder_path.rstrip("/") + "/"
         s3 = fsspec.filesystem("s3", **self.storage_options)

requirements/requirements_filters.txt

@@ -1,6 +1,6 @@
 torch
 torchvision
 autokeras
-git+https://github.com/openai/CLIP.git
+CLIP @ git+https://github.com/openai/CLIP.git
 huggingface_hub
-git+https://github.com/boomb0om/CRAFT-text-detection/
+CRAFT @ git+https://github.com/boomb0om/CRAFT-text-detection/

setup.py

@@ -0,0 +1,23 @@
+import os
+from setuptools import setup, find_packages
+
+
+def get_requirements(filename: str = 'requirements.txt'):
+    here = os.path.dirname(os.path.realpath(__file__))
+    with open(os.path.join(here, filename), 'r') as f:
+        requires = [line.replace('\n', '') for line in f.readlines()]
+    return requires
+
+
+setup(
+    name="DPF",
+    version="0.0.9",
+    description="",
+    author="Igor Pavlov, Mikhail Shoytov and Anastasia Lysenko",
+    url='https://github.com/ai-forever/DataProcessingFramework',
+    packages=find_packages(include=['DPF*']),
+    install_requires=get_requirements(),
+    extras_require={
+        "filters": get_requirements(os.path.join('requirements', 'requirements_filters.txt')),
+    }
+)