[DATALAD] Recorded changes

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.csv

@@ -0,0 +1 @@
+../../.git/annex/objects/mm/76/MD5E-s8726203--b424bd0f2655a2cac2b8bee51ba8eae4.csv/MD5E-s8726203--b424bd0f2655a2cac2b8bee51ba8eae4.csv

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.nc

@@ -0,0 +1 @@
+../../.git/annex/objects/g4/30/MD5E-s2671601--fc580db52a3a458fcc18ee148dbb446a.nc/MD5E-s2671601--fc580db52a3a458fcc18ee148dbb446a.nc

extracted_data/v2024-11-14/FAOSTAT_Agrifood_system_emissions_v2024-11-14.yaml

@@ -0,0 +1,23 @@
+attrs:
+  area: area (ISO3)
+  cat: category (IPCC2006_PRIMAP)
+  comment: Published by Food and Agriculture Organization of the United Nations (FAO),
+    converted to PRIMAP2 format by Daniel Busch
+  contact: daniel.busch@climate-resource.com
+  entity: N2O
+  institution: Food and Agriculture Organization of the United Nations
+  references: https://www.fao.org/faostat
+  rights: Creative Commons Attribution-4.0 International licence (CC BY 4.0)
+  scen: scenario (FAO)
+  title: Agrifood systems emissions
+data_file: FAOSTAT_Agrifood_system_emissions_v2024-11-14.csv
+dimensions:
+  '*':
+  - area (ISO3)
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - scenario (FAO)
+  - source
+  - time
+  - unit
+time_format: '%Y'

src/faostat_data_primap/read.py

@@ -248,7 +248,7 @@ def read_data(  # noqa: PLR0915 PLR0912
         output_folder.mkdir()
 
     filepath = output_folder / (output_filename + ".csv")
-    print(f"Writing primap2 file to {filepath}")
+    print(f"Writing raw primap2 file to {filepath}")
     pm2.pm2io.write_interchange_format(
         filepath,
         data_if,

tests/unit/test_conversion.py

@@ -35,7 +35,7 @@ def test_conversion_from_FAO_to_IPCC2006_PRIMAP():
     gases = ["CO2", "CH4", "N2O"]
     for var in gases:
         conv[var] = cc.Conversion.from_csv(
-            f"conversion_FAO_IPPCC2006_PRIMAP_{var}.csv", cats=cats
+            f"../../conversion_FAO_IPPCC2006_PRIMAP_{var}.csv", cats=cats
         )
 
     # ds_if = ds.pr.to_interchange_format()
@@ -130,32 +130,27 @@ def test_conversion_from_FAO_to_IPCC2006_PRIMAP():
     result_proc = result.pr.add_aggregates_coordinates(agg_info=agg_info)
 
     result_proc_if = result_proc.pr.to_interchange_format()
-    assert result_proc_if
-
-    # df_all = pd.concat([ds_if, result_if], axis=0, join="outer", ignore_index=True)
-    #
-    # compare = df_all.loc[
-    #     df_all["entity"] == "CO2"
-    #     # (df_all["category (IPCC2006_PRIMAP)"] == "3.A")
-    #     # | (df_all["category (FAOSTAT)"] == "3")
-    #     ].sort_values(by="area (ISO3)")
-    #
-    # compare_short = compare[
-    #     [
-    #         "source",
-    #         "scenario (FAO)",
-    #         "area (ISO3)",
-    #         "entity",
-    #         "unit",
-    #         "category (FAOSTAT)",
-    #         "2021",
-    #         "2022",
-    #         "2023",
-    #         "category (IPCC2006_PRIMAP)",
-    #     ]
-    # ]
-    #
-    # assert compare_short
+
+    # save raw data
+    release_name = "v2024-11-14"
+    output_filename = f"FAOSTAT_Agrifood_system_emissions_{release_name}"
+    output_folder = extracted_data_path / release_name
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+
+    filepath = output_folder / (output_filename + ".csv")
+    print(f"Writing processed primap2 file to {filepath}")
+    pm2.pm2io.write_interchange_format(
+        filepath,
+        result_proc_if,
+    )
+
+    compression = dict(zlib=True, complevel=9)
+    encoding = {var: compression for var in result_proc.data_vars}
+    filepath = output_folder / (output_filename + ".nc")
+    print(f"Writing netcdf file to {filepath}")
+    result_proc.pr.to_netcdf(filepath, encoding=encoding)
 
 
 def test_read(tmp_path):