diff --git a/item/historical/T018.py b/item/historical/T018.py new file mode 100644 index 00000000..dc2769df --- /dev/null +++ b/item/historical/T018.py @@ -0,0 +1,189 @@ +import pandas as pd +import yaml +import os + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-FRA-004(2).csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal_df_data.csv') + os.remove("internal_df_data.csv") + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + +# Execute transformation for sheet 'TAS-FRA-004(2)' +SHEET_NAME = 'TAS-FRA-004(2)' +OUTPUT_FILE_NAME = "output_data_TAS-FRA-004(2).csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() + + diff --git a/item/historical/T019.py b/item/historical/T019.py new file mode 100644 index 00000000..31c5be26 --- /dev/null +++ b/item/historical/T019.py @@ -0,0 +1,193 @@ +import os + +import pandas as pd +import yaml + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-FRA-007(3).csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal_df_data.csv') + os.remove("internal_df_data.csv") + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + + + +# Execute transformation for sheet 'TAS-FRA-007(3)' +SHEET_NAME = 'TAS-FRA-007(3)' +OUTPUT_FILE_NAME = "output_data_TAS-FRA-007(3).csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() + + + diff --git a/item/historical/T020.py b/item/historical/T020.py new file mode 100644 index 00000000..61102fd2 --- /dev/null +++ b/item/historical/T020.py @@ -0,0 +1,192 @@ +import os + +import pandas as pd +import yaml + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-VEP-018.csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + # Read the files from the main folder + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal_df_data.csv') + os.remove("internal_df_data.csv") + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + + +# Execute transformation for sheet 'TAS-VEP-018' +SHEET_NAME = 'TAS-VEP-018' +OUTPUT_FILE_NAME = "output_data_TAS-VEP-018.csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() + + diff --git a/item/serah/T018-TAS-FRA-004(2).py b/item/serah/T018-TAS-FRA-004(2).py new file mode 100644 index 00000000..377b69b6 --- /dev/null +++ b/item/serah/T018-TAS-FRA-004(2).py @@ -0,0 +1,185 @@ +import pandas as pd +import yaml + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-FRA-004(2).csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal/internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal/internal_df_data.csv') + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + +# Execute transformation for sheet 'TAS-FRA-004(2)' +SHEET_NAME = 'TAS-FRA-004(2)' +OUTPUT_FILE_NAME = "output_data_TAS-FRA-004(2).csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() \ No newline at end of file diff --git a/item/serah/T019-TAS-FRA-007(3).py b/item/serah/T019-TAS-FRA-007(3).py new file mode 100644 index 00000000..f7ebb0cc --- /dev/null +++ b/item/serah/T019-TAS-FRA-007(3).py @@ -0,0 +1,190 @@ +import pandas as pd +import yaml + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-FRA-007(3).csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal/internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal/internal_df_data.csv') + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + + + +# Execute transformation for sheet 'TAS-FRA-007(3)' +SHEET_NAME = 'TAS-FRA-007(3)' +OUTPUT_FILE_NAME = "output_data_TAS-FRA-007(3).csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() + + + diff --git a/item/serah/T020-TAS-VEP-018.py b/item/serah/T020-TAS-VEP-018.py new file mode 100644 index 00000000..d565c5d0 --- /dev/null +++ b/item/serah/T020-TAS-VEP-018.py @@ -0,0 +1,188 @@ +import pandas as pd +import yaml + +EXCEL_SHEET_NAME = "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS))2023.xlsx" +REGION_YAML_FILE_NAME = "regions_roadmap.yaml" +RULEBOOK_YAML_FILE_NAME = "sources.yaml" +OUTPUT_FILE_NAME = "output_data_TAS-VEP-018.csv" + + +class ItemTransformer: + + def __init__(self, sheetname, file_name): + self.sheetname = sheetname + self.file_name = file_name + + def execute(self): + # Read the files from the main folder + excel_file = pd.ExcelFile(EXCEL_SHEET_NAME) + region_yaml_data = self.load_yaml_to_cache(REGION_YAML_FILE_NAME) + rule_book_yaml_data = self.load_yaml_to_cache(RULEBOOK_YAML_FILE_NAME) + for sheet_name in excel_file.sheet_names: + if sheet_name in [self.sheetname]: + df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None, index_col=None) + df = self.drop_empty_data(df) + df_metadata_columns = self.load_metadata(df) + df_data = self.load_data(df) + df_data.to_csv('internal/internal_df_data.csv', index=False, header=False) + meta_data, applicable_rule = self.transform_metatdata(df_metadata_columns, rule_book_yaml_data) + df_metadata_transformed = pd.DataFrame(meta_data) + df_data_csv = pd.read_csv('internal/internal_df_data.csv') + self.create_dummy_merge_column(df_metadata_transformed, df_data_csv) + merged_df = pd.merge(df_metadata_transformed, df_data_csv, on='merge_column', how='right') + merged_df.drop('merge_column', axis=1, inplace=True) + self.transformation_data(region_yaml_data, merged_df, applicable_rule) + merged_df.to_csv(self.file_name, index=False) + print("Execution completed for sheet : "+ self.sheetname) + """ ---------------------------------------------- """ + + def load_metadata(self, df): + df_metadata_rows = df.iloc[:8, :] + df_metadata_columns = df_metadata_rows.iloc[:, [0, 1]] + return df_metadata_columns + + def load_data(self, df): + df_data_rows = df.iloc[12:64, :] + df_data = df_data_rows.iloc[:, 0:35] + return df_data + + def transform_metatdata(self, df_metadata_columns, rule_book_yaml_data): + + meta_data = {} + source_value = None + id_value = None + applicable_rule = None + indicator = "" + for row in df_metadata_columns._values: + if row[0] == "Indicator ATO Code:": + source_value = [row[1]] + elif row[0] == "Mode:": + mode_value = [row[1]] + elif row[0] == "Indicator:": + key, value = self.find_matching_rule_by_indicator(row[1], rule_book_yaml_data) + id_value = key + indicator = row[1] + applicable_rule = value + meta_data["Source"] = [applicable_rule["Source Prefix"] + " " + source_value[0]] + meta_data["Variable"] = applicable_rule["Variable"] + meta_data["Unit"] = applicable_rule["Unit"] + meta_data["Service"] = applicable_rule["Service"] + meta_data["Mode"] = applicable_rule["Mode"] + meta_data["Vehicle Type"] = self.get_vehicle_type(meta_data["Mode"], indicator, applicable_rule) + meta_data["Technology"] = ["All"] + meta_data["Fuel"] = ["All"] + meta_data["ID"] = id_value + return meta_data, applicable_rule + + def transformation_data(self, yaml_data, merged_df, applicable_rule): + self.rename_and_reorder_columns(merged_df) + unit = applicable_rule["Unit Factor"] + for column_name in merged_df.columns: + no_spaces = column_name.replace(" ", "") + try: + if not no_spaces.isalpha(): + integer_number = int(column_name.split('.')[0]) + if 1900 <= integer_number <= 2022: + merged_df[column_name] = merged_df[column_name] / unit + merged_df.rename(columns={column_name: integer_number}, inplace=True) + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + self.add_region_from_iso_code(column_name, merged_df, yaml_data) + self.fill_years(merged_df, applicable_rule) + + + def rename_and_reorder_columns(self, merged_df): + merged_df.rename(columns={'Economy Code': 'ISO Code'}, inplace=True) + merged_df.rename(columns={'Economy Name': 'Country'}, inplace=True) + # Specify the column to be shifted + country_to_shift = 'Country' + ISO_code_to_shift = 'ISO Code' + country_position = 1 # Specify the desired position (index) where the column should be moved + ISO_position = 2 # Specify the desired position (index) where the column should be moved + # Shift the column to the desired position + merged_df.insert(country_position, country_to_shift, merged_df.pop(country_to_shift)) + merged_df.insert(ISO_position, ISO_code_to_shift, merged_df.pop(ISO_code_to_shift)) + if "Remarks" in merged_df.columns: + merged_df.pop("Remarks") + + def add_region_from_iso_code(self, column_name, merged_df, yaml_data): + if column_name == "ISO Code": + region_list = [] + economy_code = merged_df[column_name] + for code in economy_code: + if code == "nan": + region_list.append("Not Found") + match_found = False + for key, value in yaml_data.items(): + countries_economy_code = value["countries"] + if code in countries_economy_code: + match_found = True + region_list.append(key) + break + if not match_found: + region_list.append("Not Found") + merged_df.insert(3, "Region", region_list) + + def fill_years(self, merged_df, applicable_rule): + fill_until_year = 0000 + fill_year_from_index = 0 + for index, column_name in enumerate(merged_df.columns): + try: + if isinstance(column_name, int): + fill_until_year = column_name + fill_year_from_index = index + break + except Exception as e: + print(f"There is a exception transforming column : {column_name}") + fill_years_from = applicable_rule["Fill years from"] + + i = 0 + number_of_years_to_fill = fill_until_year - fill_years_from + while i < number_of_years_to_fill: + merged_df.insert(fill_year_from_index, fill_years_from, None) + fill_years_from += 1 + i += 1 + fill_year_from_index += 1 + + + def create_dummy_merge_column(self, df_metadata_tranformed, df_data_csv): + df_metadata_tranformed['merge_column'] = 1 + df_data_csv["merge_column"] = 1 + + def drop_empty_data(self, df): + # Drop empty columns and unnecessary rows + return df.dropna(axis=1, how='all').dropna(axis=0, how='all') + + def load_yaml_to_cache(self, filename): + yaml_file_path = f"config/{filename}" + with open(yaml_file_path, 'r') as file: + yaml_data_cache = yaml.safe_load(file) + return yaml_data_cache + + def find_matching_rule_by_indicator(self, indicator, rule_book_yaml_data): + match_found = False + for key, inner_dict in rule_book_yaml_data.items(): + if inner_dict["Name"] in indicator: + match_found = True + return key, inner_dict + if not match_found: + print("Matching rule not found for the indicator present in your sheet") + + def get_vehicle_type(self, mode, indicator, applicable_rule): + generated_vehicle_type = mode[0] + " " + indicator.split(' -')[0].lower() + try: + rule_vehicle_type_value = applicable_rule["Vehicle Type"][generated_vehicle_type] + return rule_vehicle_type_value + except KeyError as e: + rule_vehicle_type_value = applicable_rule["Vehicle Type"]["Default"] + return rule_vehicle_type_value + + +# Execute transformation for sheet 'TAS-VEP-018' +SHEET_NAME = 'TAS-VEP-018' +OUTPUT_FILE_NAME = "output_data_TAS-VEP-018.csv" +item_transformer = ItemTransformer(SHEET_NAME, OUTPUT_FILE_NAME) +""" ---------------------------------------------- """ +print("Starting execution for sheet : "+ item_transformer.sheetname) +item_transformer.execute() + diff --git a/item/serah/__init__.py b/item/serah/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/item/serah/config/__init__.py b/item/serah/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/item/serah/config/regions_roadmap.yaml b/item/serah/config/regions_roadmap.yaml new file mode 100644 index 00000000..3e87eb23 --- /dev/null +++ b/item/serah/config/regions_roadmap.yaml @@ -0,0 +1,287 @@ +# iTEM2 (generated from map_cntry_region_ALL.csv) +Africa: + countries: + - AGO + - BDI + - BEN + - BFA + - BWA + - CAF + - CIV + - CMR + - COD + - COG + - COM + - CPV + - DJI + - DZA + - ERI + - ESH + - ETH + - GAB + - GHA + - GIN + - GMB + - GNB + - GNQ + - KEN + - LBR + - LBY + - LSO + - MAR + - MDG + - MLI + - MOZ + - MRT + - MUS + - MWI + - NAM + - NER + - NGA + - REU + - RWA + - SDN + - SEN + - SLE + - SOM + - STP + - SWZ + - TCD + - TGO + - TUN + - TZA + - UGA + - ZAF + - ZMB + - ZWE + +Australia: + countries: + - AUS + +Brazil: + countries: + - BRA + +Canada: + countries: + - CAN + +China: + countries: + - CHN + - HKG + - MAC + - TWN + +EU-27: + countries: + - AUT + - BEL + - BGR + - CYP + - CZE + - DEU + - DNK + - ESP + - EST + - FIN + - FRA + - GBR + - GRC + - GRL + - HRV + - HUN + - IRL + - ITA + - LTU + - LUX + - LVA + - MLT + - NLD + - POL + - PRT + - ROM + - ROU + - SVK + - SVN + - SWE + +India: + countries: + - IND + +Japan: + countries: + - JPN + +Mexico: + countries: + - MEX + +Middle East: + countries: + - ARE + - BHR + - EGY + - IRN + - IRQ + - ISR + - JOR + - KWT + - LBN + - OMN + - PSE + - QAT + - SAU + - SYR + - YEM + +Non-EU Europe: + countries: + - ALB + - AND + - ARM + - AZE + - BIH + - BLR + - CHE + - CHI + - FRO + - GEO + - GIB + - IMN + - ISL + - LIE + - MCO + - MDA + - MKD + - MNE + - NOR + - SCG + - SHN + - SJM + - SMR + - SPM + - SRB + - TCA + - TUR + - UKR + - VAT + - WLF + - YUG + +Other Asia-Pacific: + countries: + - AFG + - ASM + - BGD + - BRN + - BTN + - CCK + - COK + - CXR + - FJI + - FSM + - GUM + - IDN + - KAZ + - KGZ + - KHM + - KIR + - LAO + - LKA + - MDV + - MHL + - MMR + - MNG + - MNP + - MYS + - MYT + - NCL + - NFK + - NIU + - NPL + - NRU + - NZL + - PAK + - PCI + - PCN + - PHL + - PLW + - PNG + - PRK + - PYF + - SGP + - SLB + - SYC + - THA + - TJK + - TKL + - TKM + - TLS + - TON + - TUV + - UZB + - VNM + - VUT + - WSM + +Other Latin America: + countries: + - ABW + - AIA + - ANT + - ARG + - ATG + - BHS + - BLZ + - BMU + - BOL + - BRB + - CHL + - COL + - CRI + - CUB + - CYM + - DMA + - DOM + - ECU + - FLK + - GLP + - GRD + - GTM + - GUF + - GUY + - HND + - HTI + - JAM + - KNA + - LCA + - MSR + - MTQ + - NIC + - PAN + - PER + - PRY + - SLV + - SUR + - TTO + - URY + - VCT + - VEN + - VGB + - VIR + +Russia: + countries: + - RUS + +South Korea: + countries: + - KOR + +U.S.: + countries: + - PRI + - USA \ No newline at end of file diff --git a/item/serah/config/sources.yaml b/item/serah/config/sources.yaml new file mode 100644 index 00000000..808b498a --- /dev/null +++ b/item/serah/config/sources.yaml @@ -0,0 +1,55 @@ +T018: + Name: Freight Transport - Tonne-km for Roads + Variable: Freight Activity + Service: Freight + Unit: 10^9 tonne-km / yr + Unit Factor: 1000 + Source Prefix: ATO2023 + Mode: Road + Vehicle Type: + Road freight transport: All + Road freight transport for hire and reward: For Hire and Reward + Road freight transport on own account: For Own Account + Rail freight transport: All + Pipelines transport: Pipeline + Inland waterways freight transport: Inland + Total inland freight transport: All + Inland (exl. Pipeline): All + Default: All + Technology: All + Fuel: All + Fill years from: 1950 +T019: + Name: Freight Transport - Tonne-km for Aviation (Domestic) + Variable: Freight Activity + Service: Freight + Unit: 10^9 tonne-km / yr + Unit Factor: 1000 + Source Prefix: ATO2023 + Mode: Aviation (Domestic) + Vehicle Type: + Total inland passenger transport is: All + Rail passenger transport is: All + Road passenger transport by passenger cars is: LDV + Road passenger transport by buses and coaches is: Bus + Default: All + Technology: All + Fuel: All + Fill years from: 1950 +T020: + Name: Vehicle registration (Bus) + Variable: Stock + Service: Passenger + Unit: 10^6 vehicle + Unit Factor: 1000000 + Source Prefix: ATO2023 + Mode: Road + Vehicle Type: + Total inland passenger transport is: All + Rail passenger transport is: All + Road passenger transport by passenger cars is: LDV + Road passenger transport by buses and coaches is: Bus + Default: Bus + Technology: All + Fuel: All + Fill years from: 1950 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..5c33692b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +pandas~=2.1.3 +click~=8.1.7 +plotnine~=0.12.4 +Pint~=0.22 +PyYAML~=6.0.1 +numpy~=1.26.2 +xarray~=2023.11.0 +pycountry~=22.3.5 +requests~=2.31.0 +setuptools~=65.5.1 +packaging~=23.2 \ No newline at end of file