{"id":301428,"name":"CRA5","description":"Extreme Compression of ERA5 for Portable Global Climate and Weather Research via an Efficient Variational Transformer.","url":"https://github.com/taohan10200/cra5","last_synced_at":"2026-05-13T20:30:31.578Z","repository":{"id":246772406,"uuid":"794439281","full_name":"taohan10200/CRA5","owner":"taohan10200","description":"A large compression model for weather and climate data, which compresses a 400+ TB ERA5 dataset into a new 0.8 TB CRA5 dataset.","archived":false,"fork":false,"pushed_at":"2024-10-19T13:17:17.000Z","size":16167,"stargazers_count":93,"open_issues_count":6,"forks_count":10,"subscribers_count":5,"default_branch":"main","last_synced_at":"2026-04-05T02:09:07.399Z","etag":null,"topics":["auto-encoder","data-compression","era5","numerical-weather-forecasting"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/taohan10200.png","metadata":{"files":{"readme":"Readme.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2024-05-01T06:54:56.000Z","updated_at":"2026-03-24T20:58:42.000Z","dependencies_parsed_at":null,"dependency_job_id":"e96180eb-ad9a-4a52-be3b-db23547b5292","html_url":"https://github.com/taohan10200/CRA5","commit_stats":{"total_commits":55,"total_committers":1,"mean_commits":55.0,"dds":0.0,"last_synced_commit":"2b9e06d8ca31f7b27c5039c9b5fc3334a43b4976"},"previous_names":["taohan10200/cra5"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/taohan10200/CRA5","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200","download_url":"https://codeload.github.com/taohan10200/CRA5/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":31669117,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-04-10T17:19:37.612Z","status":"online","status_checked_at":"2026-04-11T02:00:05.776Z","response_time":54,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"taohan10200","name":"tao han","uuid":"46162738","kind":"user","description":"","email":"","website":null,"location":null,"twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/46162738?u=cb38103d1f37a0617b32a83c4d3d9a6804c15a1f\u0026v=4","repositories_count":4,"last_synced_at":"2023-03-10T11:46:18.535Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/taohan10200","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2023-03-10T11:46:18.536Z","updated_at":"2023-03-10T11:46:18.536Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200/repositories"},"packages":[{"id":10525320,"name":"cra5","ecosystem":"pypi","description":"A large compression model for weather and climate data, which compresses a 200+ TB ERA5 dataset into a new 0.7TB CRA5 dataset.","homepage":"https://github.com/taohan10200/CRA5","licenses":"BSD 3-Clause Clear License","normalized_licenses":["BSD-3-Clause-Clear"],"repository_url":"https://github.com/taohan10200/CRA5","keywords_array":[],"namespace":null,"versions_count":6,"first_release_published_at":"2024-07-02T03:08:24.000Z","latest_release_published_at":"2024-07-04T02:54:04.000Z","latest_release_number":"0.0.3.dev1","last_synced_at":"2026-04-13T06:07:03.499Z","created_at":"2024-07-02T03:16:37.725Z","updated_at":"2026-04-13T06:07:03.499Z","registry_url":"https://pypi.org/project/cra5/","install_command":"pip install cra5 --index-url https://pypi.org/simple","documentation_url":"https://cra5.readthedocs.io/","metadata":{"funding":null,"documentation":null,"classifiers":["Development Status :: 3 - Alpha","Intended Audience :: Developers","Intended Audience :: Science/Research","License :: OSI Approved :: BSD License","Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.8","Programming Language :: Python :: 3.9","Topic :: Scientific/Engineering :: Artificial Intelligence"],"normalized_name":"cra5","project_status":null},"repo_metadata":{"id":246772406,"uuid":"794439281","full_name":"taohan10200/CRA5","owner":"taohan10200","description":"A large compression model for weather and climate data, which compresses a 400+ TB ERA5 dataset into a new 0.8 TB CRA5 dataset.","archived":false,"fork":false,"pushed_at":"2024-10-19T13:17:17.000Z","size":16167,"stargazers_count":21,"open_issues_count":4,"forks_count":1,"subscribers_count":2,"default_branch":"main","last_synced_at":"2024-10-20T11:17:35.523Z","etag":null,"topics":["auto-encoder","data-compression","era5","numerical-weather-forecasting"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/taohan10200.png","metadata":{"files":{"readme":"Readme.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2024-05-01T06:54:56.000Z","updated_at":"2024-10-19T13:17:21.000Z","dependencies_parsed_at":null,"dependency_job_id":"e96180eb-ad9a-4a52-be3b-db23547b5292","html_url":"https://github.com/taohan10200/CRA5","commit_stats":{"total_commits":54,"total_committers":1,"mean_commits":54.0,"dds":0.0,"last_synced_commit":"0a7a2418b5b0b78067b3a3de53fe10fb24fe7f24"},"previous_names":["taohan10200/cra5"],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2FCRA5/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200","download_url":"https://codeload.github.com/taohan10200/CRA5/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":220913619,"owners_count":16721639,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"},"owner_record":{"login":"taohan10200","name":"tao han","uuid":"46162738","kind":"user","description":"","email":"","website":null,"location":null,"twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/46162738?u=cb38103d1f37a0617b32a83c4d3d9a6804c15a1f\u0026v=4","repositories_count":4,"last_synced_at":"2023-03-10T11:46:18.535Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/taohan10200","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2023-03-10T11:46:18.536Z","updated_at":"2023-03-10T11:46:18.536Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/taohan10200/repositories"},"tags":[]},"repo_metadata_updated_at":"2024-10-24T08:33:12.849Z","dependent_packages_count":0,"downloads":12,"downloads_period":"last-month","dependent_repos_count":0,"rankings":{"downloads":null,"dependent_repos_count":60.2633566929498,"dependent_packages_count":10.700942664161119,"stargazers_count":null,"forks_count":null,"docker_downloads_count":null,"average":35.48214967855546},"purl":"pkg:pypi/cra5","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/pypi/cra5","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/pypi/cra5","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/pypi/cra5/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":{"last_synced_at":"2024-10-24T08:30:14.680Z","issues_count":8,"pull_requests_count":0,"avg_time_to_close_issue":2071316.0,"avg_time_to_close_pull_request":null,"issues_closed_count":2,"pull_requests_closed_count":0,"pull_request_authors_count":0,"issue_authors_count":6,"avg_comments_per_issue":3.0,"avg_comments_per_pull_request":null,"merged_pull_requests_count":0,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":8,"past_year_pull_requests_count":0,"past_year_avg_time_to_close_issue":2071316.0,"past_year_avg_time_to_close_pull_request":null,"past_year_issues_closed_count":2,"past_year_pull_requests_closed_count":0,"past_year_pull_request_authors_count":0,"past_year_issue_authors_count":6,"past_year_avg_comments_per_issue":3.0,"past_year_avg_comments_per_pull_request":null,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":0,"issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2Fcra5/issues","maintainers":[],"active_maintainers":[]},"versions_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/cra5/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/cra5/version_numbers","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/cra5/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/cra5/related_packages","codemeta_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages/cra5/codemeta","maintainers":[{"uuid":"hantao10200","login":"hantao10200","name":null,"email":null,"url":null,"packages_count":1,"html_url":"https://pypi.org/user/hantao10200/","role":null,"created_at":"2024-07-02T03:16:39.782Z","updated_at":"2024-07-02T03:16:39.782Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/maintainers/hantao10200/packages"}],"registry":{"name":"pypi.org","url":"https://pypi.org","ecosystem":"pypi","default":true,"packages_count":841573,"maintainers_count":359517,"namespaces_count":0,"keywords_count":271316,"github":"pypi","metadata":{"funded_packages_count":53172},"icon_url":"https://github.com/pypi.png","created_at":"2022-04-04T15:19:23.364Z","updated_at":"2026-04-09T05:08:03.587Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/packages","maintainers_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/maintainers","namespaces_url":"https://packages.ecosyste.ms/api/v1/registries/pypi.org/namespaces"}}],"commits":{"id":2375317,"full_name":"taohan10200/cra5","default_branch":"main","total_commits":55,"total_committers":1,"total_bot_commits":0,"total_bot_committers":0,"mean_commits":55.0,"dds":0.0,"past_year_total_commits":0,"past_year_total_committers":0,"past_year_total_bot_commits":0,"past_year_total_bot_committers":0,"past_year_mean_commits":0.0,"past_year_dds":0.0,"last_synced_at":"2026-04-13T06:00:33.178Z","last_synced_commit":"2b9e06d8ca31f7b27c5039c9b5fc3334a43b4976","created_at":"2024-10-07T00:10:14.473Z","updated_at":"2026-04-13T06:00:33.135Z","committers":[{"name":"taohan10200","email":"taohan10200@163.com","login":"taohan10200","count":55}],"past_year_committers":[],"commits_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2Fcra5/commits","host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-04-15T00:00:09.512Z","repositories_count":6213593,"commits_count":903370819,"contributors_count":34924210,"owners_count":1144562,"icon_url":"https://github.com/github.png","host_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories"}},"issues_stats":{"full_name":"taohan10200/cra5","html_url":"https://github.com/taohan10200/cra5","last_synced_at":"2026-04-09T04:00:55.435Z","status":"error","issues_count":11,"pull_requests_count":0,"avg_time_to_close_issue":5523648.75,"avg_time_to_close_pull_request":null,"issues_closed_count":4,"pull_requests_closed_count":0,"pull_request_authors_count":0,"issue_authors_count":9,"avg_comments_per_issue":1.7272727272727273,"avg_comments_per_pull_request":null,"merged_pull_requests_count":0,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":1,"past_year_pull_requests_count":0,"past_year_avg_time_to_close_issue":null,"past_year_avg_time_to_close_pull_request":null,"past_year_issues_closed_count":0,"past_year_pull_requests_closed_count":0,"past_year_pull_request_authors_count":0,"past_year_issue_authors_count":1,"past_year_avg_comments_per_issue":1.0,"past_year_avg_comments_per_pull_request":null,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":0,"created_at":"2024-10-07T00:10:14.861Z","updated_at":"2026-04-09T04:00:55.435Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2Fcra5","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/taohan10200%2Fcra5/issues","issue_labels_count":{},"pull_request_labels_count":{},"issue_author_associations_count":{"NONE":11},"pull_request_author_associations_count":{},"issue_authors":{"px39n":2,"tung-nd":2,"0rhisia0":1,"salmanmohebi":1,"siddevkota":1,"Mapirlet":1,"gerome-andry":1,"Sardingfish":1,"vitusbenson":1},"pull_request_authors":{},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-04-13T00:00:15.727Z","repositories_count":14240319,"issues_count":34631349,"pull_requests_count":113115597,"authors_count":11233269,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{},"past_year_pull_request_labels_count":{},"past_year_issue_author_associations_count":{"NONE":1},"past_year_pull_request_author_associations_count":{},"past_year_issue_authors":{"salmanmohebi":1},"past_year_pull_request_authors":{},"maintainers":[],"active_maintainers":[]},"events":{"total":{"ForkEvent":6,"IssuesEvent":7,"WatchEvent":48,"IssueCommentEvent":6,"PushEvent":1},"last_year":{"ForkEvent":5,"IssuesEvent":1,"WatchEvent":14,"IssueCommentEvent":1}},"keywords":["auto-encoder","data-compression","era5","numerical-weather-forecasting"],"dependencies":[],"score":7.160069207596127,"created_at":"2024-10-07T00:10:11.065Z","updated_at":"2026-05-13T20:30:31.601Z","avatar_url":"https://github.com/taohan10200.png","language":"Python","category":"Atmosphere","sub_category":"Meteorological Observation and Forecast","monthly_downloads":12,"total_dependent_repos":0,"total_dependent_packages":0,"readme":"\n\u003c!-- ![ID-CompressAI-logo](assets/CRA5LOGO.svg =750x140) --\u003e\n\u003ca href=\"url\"\u003e\u003cimg src=\"assets/CRA5LOGO.svg\" align=\"center\"\u003e\u003c/a\u003e\n\n[![License](https://img.shields.io/github/license/InterDigitalInc/CompressAI?color=blue)](https://github.com/InterDigitalInc/CompressAI/blob/master/LICENSE)\n[![PyPI](https://img.shields.io/pypi/v/cra5?color=brightgreen)](https://pypi.org/project/compressai/)\n[![Downloads](https://pepy.tech/badge/cra5)](https://pypi.org/project/cra5/#files)\n\n[Paper:CRA5: Extreme Compression of ERA5 for Portable Global Climate and Weather Research via an Efficient Variational Transformer](https://arxiv.org/abs/2405.03376)\n\n# Introduction and get started\n### CRA5 dataset now is available at [OneDrive](https://hkustconnect-my.sharepoint.com/:f:/g/personal/thanad_connect_ust_hk/EjGupNV_IcZKre1PdN65dFUB2sBOHX3MCH4cycBtcSm2yA?e=GAqS1J)\nCRA5 is a extreme **compressed weather dataset** of the most popular ERA5 reanalysis dataset. The repository also includes **compression models**, **forecasting model** for researchers to conduct portable weather and climate research.\n\nCRA5 currently provides:\n\n* A customized variaitional transformer (VAEformer) for climate data compression\n* A dataset CRA5 less than 1 TiB, but contains the same information with 400+ TiB ERA5 dataset. Covering houly ERA5 from year 1979 to 2023.    \n* A pre-trained Auto-Encoder on the climate/weather data to support some potential weather research.\n\n\n\n\u003e **Note**: Multi-GPU support is now experimental.\n\n## Installation\n\nCRA5 supports python 3.8+ and PyTorch 1.7+.\n```\nconda create --name cra5 python=3.10 -y \nconda activate cra5\n```\n\n\n**Please install cra5 from source**:\n\nA C++17 compiler, a recent version of pip (19.0+), and common python packages are also required (see `setup.py` for the full list).\n\nTo get started locally and install the development version of CRA5, run the following commands in a [virtual environment](https://docs.python.org/3.6/library/venv.html):\n\n```bash\ngit clone https://github.com/taohan10200/CRA5\ncd CRA5\n\npip install -U pip \u0026\u0026 pip install -e .\n```\n\n\u003c!-- For a custom installation, you can also run one of the following commands:\n* `pip install -e '.[dev]'`: install the packages required for development (testing, linting, docs)\n* `pip install -e '.[tutorials]'`: install the packages required for the tutorials (notebooks)\n* `pip install -e '.[all]'`: install all the optional packages --\u003e\n\n\n\n\u003c!-- ## Documentation --\u003e\n\n\u003c!-- * [Installation](https://interdigitalinc.github.io/CompressAI/installation.html)\n* [CompressAI API](https://interdigitalinc.github.io/CompressAI/)\n* [Training your own model](https://interdigitalinc.github.io/CompressAI/tutorials/tutorial_train.html)\n* [List of available models (model zoo)](https://interdigitalinc.github.io/CompressAI/zoo.html) --\u003e\n\n# Test\n```\npython test.py\n``` \n\n# Usages\n## Using with API: \n\u003e Supporting functions like: Compression / decompression / latents representation / feature visulization / reconstructed visulization\n```python\n# We build a downloader to help use download the original ERA5 netcdf files for testing.\n\n# data/ERA5/2024/2024-06-01T00:00:00_pressure.nc (513MiB) and data/ERA5/2024/2024-06-01T00:00:00_single.nc (18MiB) \nfrom cra5.api.era5_downloader import era5_downloader\nERA5_data = era5_downloader('./cra5/api/era5_config.py') #specify the dataset config for what we want to download\ndata = ERA5_data.get_form_timestamp(time_stamp=\"2024-06-01T00:00:00\",\n                                    local_root='./data/ERA5')\n\n# After getting the ERA5 data ready, you can explore the compression.\nfrom cra5.api import cra5_api\ncra5_API = cra5_api()\n\n####=======================compression functions=====================\n# Return a continuous latent y for ERA5 data at 2024-06-01T00:00:00\ny = cra5_API.encode_to_latent(time_stamp=\"2024-06-01T00:00:00\") \n\n# Return the the arithmetic coded binary stream of y \nbin_stream = cra5_API.latent_to_bin(y=y)  \n\n# Or if you want to directly compress and save the binary stream to a folder\ncra5_API.encode_era5_as_bin(time_stamp=\"2024-06-01T00:00:00\", save_root='./data/cra5')  \n\n\n####=======================decompression functions=====================\n# Starting from the bin_stream, you can decode the binary file to the quantized latent.\ny_hat = cra5_API.bin_to_latent(bin_path=\"./data/CRA5/2024/2024-06-01T00:00:00.bin\")  # Decoding from binary can only get the quantized latent.\n\n# Return the normalized cra5 data\nnormlized_x_hat = cra5_API.latent_to_reconstruction(y_hat=y_hat) \n\n\n# If you have saveed  or downloaded the binary file, then you can directly restore the binary file into reconstruction.\nnormlized_x_hat = cra5_API.decode_from_bin(\"2024-06-01T00:00:00\", return_format='normalized') # Return the normalized cra5 data\nx_hat = cra5_API.decode_from_bin(\"2024-06-01T00:00:00\", return_format='de_normalized') # Return the de-normalized cra5 data\n\n# Show some channels of the latent\ncra5_API.show_latent(\n\tlatent=y_hat.squeeze(0).cpu().numpy(), \n\ttime_stamp=\"2024-06-01T00:00:00\", \n\tshow_channels=[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150],\n    save_path = './data/vis')\n\n```\n\u003c!-- ![ID-CompressAI-logo](assets/2024-06-01T00:00:00_latent.png =400x140) --\u003e\n\u003ca href=\"url\"\u003e\u003cimg src=\"assets/2024-06-01T00_latent.png\" align=\"center\"\u003e\u003c/a\u003e\n\n```python\n# show some variables for the constructed data\ncra5_API.show_image(\n\treconstruct_data=x_hat.cpu().numpy(), \n\ttime_stamp=\"2024-06-01T00:00:00\", \n\tshow_variables=['z_500', 'q_500', 'u_500', 'v_500', 't_500', 'w_500'],\n    save_path = './data/vis')\n```\n\n\u003c!-- ![ID-CompressAI-logo](assets/CRA5LOGO.svg =400x140) --\u003e\n\u003ca href=\"url\"\u003e\u003cimg src=\"assets/2024-06-01T00.png\" align=\"center\"\u003e\u003c/a\u003e\n\n\n\n### Or using with the pre-trained model\n\n```python\nimport os \nimport torch\nfrom cra5.models.compressai.zoo import vaeformer_pretrained\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(device)\nnet = vaeformer_pretrained(quality=268, pretrained=True).eval().to(device)\ninput_data_norm = torch.rand(1,268, 721,1440).to(device) #This is a proxy weather data. It actually should be a \n\nprint(x.shape)\nwith torch.no_grad():\n    out_net = net.compress(x) \n    \nprint(out_net)\n```\n\n\n\n# Features\n\n## 1. CRA5 dataset is a product of the VAEformer applied in the atmospheric science. We explore this to facilitate the research in weather and climate. \n\n* **Train the large data-driven numerical weather forecasting models with our CRA5**\n\n\u003e **Note**: For researches who do not have enough disk space to store the 300 TiB+ ERA5 dataset, but have interests to train a large weather forecasting model, like [FengWu-GHR](https://arxiv.org/abs/2402.00059),  this research can help you save it into less than 1 TiB disk.  \n\nOur preliminary attemp has proven that the CRA5 dataset can train the very very similar NWP model compared with the original ERA5 dataset. Also, with this dataset, you can easily train a Nature published forecasting model, like [Pangu-Weather](https://www.nature.com/articles/s41586-023-06185-3). \n\n\u003c!-- ![ID-CompressAI-logo](assets/rmse_acc_bias_activity.png =400x140) --\u003e\n\u003ca href=\"url\"\u003e\u003cimg src=\"assets/rmse_acc_bias_activity.png\" align=\"center\"\u003e\u003c/a\u003e\n\n## 2. VAEformer is a powerful compression model, we hope it can be extended to other domains, like image and video compression.\n\n\u003c!-- ![ID-CompressAI-logo](assets/MSE_supp_new.png =400x140) --\u003e\n\u003ca href=\"url\"\u003e\u003cimg src=\"assets/MSE_supp_new.png\" align=\"center\"\u003e\u003c/a\u003e\n\n\n\n## 3 VAEformer is based on the Auto-Encoder-Decoder, we provide a pretrained VAE for the weather research, you can use our VAEformer to get the latents for downstream research, like diffusion-based or other generation-based forecasting methods.\n\n* **Using it as a Auto-Encoder-Decoder**\n\n\u003e **Note**: For people who are intersted in diffusion-based or other generation-based forecasting methods, we can provide an Auto Encoder and decoder for the weather research, you can use our VAEformer to get the latents for downstream research.\n\n\n\n\n\u003c!-- Script and notebook examples can be found in the `examples/` directory.\n\nTo encode/decode images with the provided pre-trained models, run the\n`codec.py` example:\n\n```bash\npython3 examples/codec.py --help\n```\n\nAn examplary training script with a rate-distortion loss is provided in\n`examples/train.py`. You can replace the model used in the training script\nwith your own model implemented within CompressAI, and then run the script for a\nsimple training pipeline:\n\n```bash\npython3 examples/train.py -d /path/to/my/image/dataset/ --epochs 300 -lr 1e-4 --batch-size 16 --cuda --save\n```\n\u003e **Note:** the training example uses a custom [ImageFolder](https://interdigitalinc.github.io/CompressAI/datasets.html#imagefolder) structure.\n\nA jupyter notebook illustrating the usage of a pre-trained model for learned image\ncompression is also provided in the `examples` directory:\n\n```bash\npip install -U ipython jupyter ipywidgets matplotlib\njupyter notebook examples/\n``` --\u003e\n\n\u003c!-- ### Evaluation\n\nTo evaluate a trained model on your own dataset, CompressAI provides an\nevaluation script:\n\n```bash\npython3 -m compressai.utils.eval_model checkpoint /path/to/images/folder/ -a $ARCH -p $MODEL_CHECKPOINT...\n```\n\nTo evaluate provided pre-trained models:\n\n```bash\npython3 -m compressai.utils.eval_model pretrained /path/to/images/folder/ -a $ARCH -q $QUALITY_LEVELS...\n```\n\nTo plot results from bench/eval_model simulations (requires matplotlib by default):\n\n```bash\npython3 -m compressai.utils.plot --help --\u003e\n\u003c!-- ``` --\u003e\n\n\u003c!-- To evaluate traditional codecs:\n\n```bash\npython3 -m compressai.utils.bench --help\npython3 -m compressai.utils.bench bpg --help\npython3 -m compressai.utils.bench vtm --help\n```\n\nFor video, similar tests can be run, CompressAI only includes ssf2020 for now:\n\n```bash\npython3 -m compressai.utils.video.eval_model checkpoint /path/to/video/folder/ -a ssf2020 -p $MODEL_CHECKPOINT...\npython3 -m compressai.utils.video.eval_model pretrained /path/to/video/folder/ -a ssf2020 -q $QUALITY_LEVELS...\npython3 -m compressai.utils.video.bench x265 --help\npython3 -m compressai.utils.video.bench VTM --help\npython3 -m compressai.utils.video.plot --help\n``` --\u003e\n\n\u003c!-- ## Tests\n\nRun tests with `pytest`:\n\n```bash\npytest -sx --cov=compressai --cov-append --cov-report term-missing tests\n```\n\nSlow tests can be skipped with the `-m \"not slow\"` option. --\u003e\n\n\n## License\n\nCompressAI is licensed under the BSD 3-Clause Clear License\n\n## Contributing\n\nWe welcome feedback and contributions. Please open a GitHub issue to report\nbugs, request enhancements or if you have any questions.\n\nBefore contributing, please read the CONTRIBUTING.md file.\n\n## Authors\n\n* Tao Han ([hantao10200@gmail.com](mailto:hantao10200@gmail.com)) \n* Zhenghao Chen.\n\n## Citation\n\nIf you use this project, please cite the relevant original publications for the models and datasets, and cite this project as:\n\n```\n@article{han2024cra5extremecompressionera5,\n      title={CRA5: Extreme Compression of ERA5 for Portable Global Climate and Weather Research via an Efficient Variational Transformer}, \n      author={Tao Han and Zhenghao Chen and Song Guo and Wanghan Xu and Lei Bai},\n      year={2024},\n      eprint={2405.03376},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG},\n      url={https://arxiv.org/abs/2405.03376}, \n}\n```\n\nFor any work related to the forecasting models, please cite\n```\n@article{han2024fengwughr,\ntitle={FengWu-GHR: Learning the Kilometer-scale Medium-range Global Weather Forecasting}, \nauthor={Tao Han and Song Guo and Fenghua Ling and Kang Chen and Junchao Gong and Jingjia Luo and Junxia Gu and Kan Dai and Wanli Ouyang and Lei Bai},\nyear={2024},\neprint={2402.00059},\narchivePrefix={arXiv},\nprimaryClass={cs.LG}\n}\n```\n\n## The weather variabls supported in CRA5 and their numerical error\n\nCRA5 contains a total of 268 variables, including 7 pressure-level variables from the [ERA5 pressure level archive](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-pressure-levels) and 9 [surface variables](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels?tab=overview) . \n\n| Variable | channel | error   | Variable | channel | error   | Variable | channel | error   |Variable | channel  | error   |Variable | channel  | error   |  \n|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|\n| geopotential  | z_1000  | 9.386     | specific_humidity  | q_1000  | 0.00033   | u_component_of_wind | u_1000  | 0.416     |v_component_of_wind  | v_1000  | 0.411     | temperature  | t_1000  | 0.405     |\n| geopotential  | z_975   | 7.857     | specific_humidity  | q_975   | 0.00032   | u_component_of_wind | u_975   | 0.448     |v_component_of_wind  | v_975   | 0.442     | temperature  | t_975   | 0.380     |\n| geopotential  | z_950   | 6.802     | specific_humidity  | q_950   | 0.00035   | u_component_of_wind | u_950   | 0.491     |v_component_of_wind  | v_950   | 0.479     | temperature  | t_950   | 0.352     |\n| geopotential  | z_925   | 6.088     | specific_humidity  | q_925   | 0.00037   | u_component_of_wind | u_925   | 0.520     |v_component_of_wind  | v_925   | 0.505     | temperature  | t_925   | 0.333     |\n| geopotential  | z_900   | 5.575     | specific_humidity  | q_900   | 0.00036   | u_component_of_wind | u_900   | 0.518     |v_component_of_wind  | v_900   | 0.503     | temperature  | t_900   | 0.321     |\n| geopotential  | z_875   | 5.259     | specific_humidity  | q_875   | 0.00035   | u_component_of_wind | u_875   | 0.517     |v_component_of_wind  | v_875   | 0.503     | temperature  | t_875   | 0.309     |\n| geopotential  | z_850   | 5.061     | specific_humidity  | q_850   | 0.00034   | u_component_of_wind | u_850   | 0.508     |v_component_of_wind  | v_850   | 0.493     | temperature  | t_850   | 0.294     |\n| geopotential  | z_825   | 4.941     | specific_humidity  | q_825   | 0.00031   | u_component_of_wind | u_825   | 0.496     |v_component_of_wind  | v_825   | 0.481     | temperature  | t_825   | 0.276     |\n| geopotential  | z_800   | 4.897     | specific_humidity  | q_800   | 0.00029   | u_component_of_wind | u_800   | 0.487     |v_component_of_wind  | v_800   | 0.472     | temperature  | t_800   | 0.259     |\n| geopotential  | z_775   | 4.947     | specific_humidity  | q_775   | 0.00027   | u_component_of_wind | u_775   | 0.486     |v_component_of_wind  | v_775   | 0.468     | temperature  | t_775   | 0.250     |\n| geopotential  | z_750   | 5.120     | specific_humidity  | q_750   | 0.00029   | u_component_of_wind | u_750   | 0.545     |v_component_of_wind  | v_750   | 0.524     | temperature  | t_750   | 0.250     |\n| geopotential  | z_700   | 5.593     | specific_humidity  | q_700   | 0.00029   | u_component_of_wind | u_700   | 0.638     |v_component_of_wind  | v_700   | 0.607     | temperature  | t_700   | 0.242     |\n| geopotential  | z_650   | 5.810     | specific_humidity  | q_650   | 0.00025   | u_component_of_wind | u_650   | 0.634     |v_component_of_wind  | v_650   | 0.610     | temperature  | t_700   | 0.242     |\n| geopotential  | z_600   | 5.882     | specific_humidity  | q_600   | 0.00020   | u_component_of_wind | u_600   | 0.633     |v_component_of_wind  | v_600   | 0.597     | temperature  | t_650   | 0.240     |\n| geopotential  | z_550   | 5.958     | specific_humidity  | q_550   | 0.00018   | u_component_of_wind | u_550   | 0.668     |v_component_of_wind  | v_550   | 0.616     | temperature  | t_600   | 0.222     |\n| geopotential  | z_500   | 6.098     | specific_humidity  | q_500   | 0.00014   | u_component_of_wind | u_500   | 0.676     |v_component_of_wind  | v_500   | 0.603     | temperature  | t_550   | 0.201     |\n| geopotential  | z_450   | 6.408     | specific_humidity  | q_450   | 0.00010   | u_component_of_wind | u_450   | 0.699     |v_component_of_wind  | v_450   | 0.649     | temperature  | t_500   | 0.185     |\n| geopotential  | z_400   | 6.851     | specific_humidity  | q_400   | 0.00007   | u_component_of_wind | u_400   | 0.733     |v_component_of_wind  | v_400   | 0.686     | temperature  | t_450   | 0.185     |\n| geopotential  | z_350   | 7.366     | specific_humidity  | q_350   | 0.00004   | u_component_of_wind | u_350   | 0.760     |v_component_of_wind  | v_350   | 0.704     | temperature  | t_400   | 0.179     |\n| geopotential  | z_300   | 8.324     | specific_humidity  | q_300   | 0.00002   | u_component_of_wind | u_300   | 0.744     |v_component_of_wind  | v_300   | 0.704     | temperature  | t_350   | 0.170     |\n| geopotential  | z_250   | 8.100     | specific_humidity  | q_250   | 0.00001   | u_component_of_wind | u_250   | 0.765     |v_component_of_wind  | v_250   | 0.701     | temperature  | t_300   | 0.160     |\n| geopotential  | z_225   | 7.698     | specific_humidity  | q_225   | 0.00001   | u_component_of_wind | u_225   | 0.722     |v_component_of_wind  | v_225   | 0.642     | temperature  | t_250   | 0.166     |\n| geopotential  | z_200   | 7.900     | specific_humidity  | q_200   | 0.00000   | u_component_of_wind | u_200   | 0.646     |v_component_of_wind  | v_200   | 0.563     | temperature  | t_225   | 0.169     |\n| geopotential  | z_175   | 8.059     | specific_humidity  | q_175   | 0.00000   | u_component_of_wind | u_175   | 0.565     |v_component_of_wind  | v_175   | 0.509     | temperature  | t_200   | 0.158     |\n| geopotential  | z_150   | 8.928     | specific_humidity  | q_150   | 0.00000   | u_component_of_wind | u_150   | 0.525     |v_component_of_wind  | v_150   | 0.458     | temperature  | t_150   | 0.149     |\n| geopotential  | z_125   | 10.813    | specific_humidity  | q_125   | 0.00000   | u_component_of_wind | u_125   | 0.479     |v_component_of_wind  | v_125   | 0.417     | temperature  | t_125   | 0.158     |\n| geopotential  | z_100   | 15.956    | specific_humidity  | q_100   | 0.00000   | u_component_of_wind | u_100   | 0.447     |v_component_of_wind  | v_100   | 0.373     | temperature  | t_100   | 0.178     |\n| geopotential  | z_70    | 11.158    | specific_humidity  | q_70    | 0.00000   | u_component_of_wind | u_70    | 0.360     |v_component_of_wind  | v_70    | 0.275     | temperature  | t_70    | 0.155     |\n| geopotential  | z_50    | 11.962    | specific_humidity  | q_50    | 0.00000   | u_component_of_wind | u_50    | 0.356     |v_component_of_wind  | v_50    | 0.242     | temperature  | t_50    | 0.158     |\n| geopotential  | z_30    | 13.317    | specific_humidity  | q_30    | 0.00000   | u_component_of_wind | u_30    | 0.348     |v_component_of_wind  | v_30    | 0.221     | temperature  | t_30    | 0.153     |\n| geopotential  | z_20    | 16.538    | specific_humidity  | q_20    | 0.00000   | u_component_of_wind | u_20    | 0.361     |v_component_of_wind  | v_20    | 0.229     | temperature  | t_20    | 0.161     |\n| geopotential  | z_10    | 19.751    | specific_humidity  | q_10    | 0.00000   | u_component_of_wind | u_10    | 0.350     |v_component_of_wind  | v_10    | 0.232     | temperature  | t_10    | 0.166     |\n| geopotential  | z_7     | 20.925    | specific_humidity  | q_7     | 0.00000   | u_component_of_wind | u_7     | 0.315     |v_component_of_wind  | v_7     | 0.225     | temperature  | t_7     | 0.161     |\n| geopotential  | z_5     | 20.825    | specific_humidity  | q_5     | 0.00000   | u_component_of_wind | u_5     | 0.307     |v_component_of_wind  | v_5     | 0.212     | temperature  | t_5     | 0.160     |\n| geopotential  | z_3     | 24.529    | specific_humidity  | q_3     | 0.00000   | u_component_of_wind | u_3     | 0.333     |v_component_of_wind  | v_3     | 0.246     | temperature  | t_3     | 0.194     |\n| geopotential  | z_2     | 28.055    | specific_humidity  | q_2     | 0.00000   | u_component_of_wind | u_2     | 0.338     |v_component_of_wind  | v_2     | 0.239     | temperature  | t_2     | 0.184     |\n| geopotential  | z_1     | 27.987    | specific_humidity  | q_1     | 0.00000   | u_component_of_wind | u_1     | 0.363     |v_component_of_wind  | v_1     | 0.245     | temperature  | t_1     | 0.182     |\n|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|--------|---------|-----------|\n| relative_humidity | r_1000  | 3.073     | vertical_velocity  w_1000  | 0.059     || 10m_v_component_of_wind  | v10     | 0.367     |\n| relative_humidity | r_975   | 3.192     | vertical_velocity  w_975   | 0.067     || 10m_u_component_of_wind  | u10     | 0.379     |\n| relative_humidity | r_950   | 3.588     | vertical_velocity  w_950   | 0.078     || 100m_v_component_of_wind  | v100    | 0.435     |\n| relative_humidity | r_925   | 3.877     | vertical_velocity  w_925   | 0.086     || 100m_u_component_of_wind  | u100    | 0.445     |\n| relative_humidity | r_900   | 3.982     | vertical_velocity  w_900   | 0.090     || 2m_temperature  | t2m     | 0.720     |\n| relative_humidity | r_875   | 4.011     | vertical_velocity  w_875   | 0.092     || total_cloud_cover  | tcc     | 0.146     |\n| relative_humidity | r_850   | 3.933     | vertical_velocity  w_850   | 0.093     || surface_pressure  | sp      | 480.222   \n| relative_humidity | r_825   | 3.789     | vertical_velocity  w_825   | 0.094     || total_precipitation  | tp1h    | 0.264     |\n| relative_humidity | r_800   | 3.555     | vertical_velocity  w_800   | 0.096     || mean_sea_level_pressure  | msl     | 12.685    |\n| relative_humidity | r_775   | 3.449     | vertical_velocity  w_775   | 0.099     |\n| relative_humidity | r_750   | 3.816     | vertical_velocity  w_750   | 0.102     |\n| relative_humidity | r_700   | 4.265     | vertical_velocity  w_700   | 0.110     |\n| relative_humidity | r_650   | 4.223     | vertical_velocity  w_650   | 0.114     |\n| relative_humidity | r_600   | 4.183     | vertical_velocity  w_600   | 0.112     |\n| relative_humidity | r_550   | 4.411     | vertical_velocity  w_550   | 0.106     |\n| relative_humidity | r_500   | 4.409     | vertical_velocity  w_500   | 0.101     |\n| relative_humidity | r_450   | 4.675     | vertical_velocity  w_450   | 0.096     |\n| relative_humidity | r_400   | 4.831     | vertical_velocity  w_400   | 0.091     |\n| relative_humidity | r_350   | 4.932     | vertical_velocity  w_350   | 0.084     |\n| relative_humidity | r_300   | 5.151     | vertical_velocity  w_300   | 0.075     |\n| relative_humidity | r_250   | 5.134     | vertical_velocity  w_250   | 0.056     |\n| relative_humidity | r_225   | 4.682     | vertical_velocity  w_225   | 0.046     |\n| relative_humidity | r_200   | 3.899     | vertical_velocity  w_200   | 0.039     |\n| relative_humidity | r_175   | 3.063     | vertical_velocity  w_175   | 0.034     |\n| relative_humidity | r_150   | 2.508     | vertical_velocity  w_150   | 0.029     |\n| relative_humidity | r_125   | 2.123     | vertical_velocity  w_125   | 0.024     |\n| relative_humidity | r_100   | 1.844     | vertical_velocity  w_100   | 0.018     |\n| relative_humidity | r_70    | 0.487     | vertical_velocity  w_70    | 0.010     |\n| relative_humidity | r_50    | 0.151     | vertical_velocity  w_50    | 0.007     |\n| relative_humidity | r_30    | 0.097     | vertical_velocity  w_30    | 0.005     |\n| relative_humidity | r_20    | 0.083     | vertical_velocity  w_20    | 0.003     |\n| relative_humidity | r_10    | 0.033     | vertical_velocity  w_10    | 0.002     |\n| relative_humidity | r_7     | 0.016     | vertical_velocity  w_7     | 0.001     |\n| relative_humidity | r_5     | 0.008     | vertical_velocity  w_5     | 0.001     |\n| relative_humidity | r_3     | 0.003     | vertical_velocity  w_3     | 0.001     |\n| relative_humidity | r_2     | 0.001     | vertical_velocity  w_2     | 0.000     |\n| relative_humidity | r_1     | 0.000     | vertical_velocity  w_1     | 0.000     |\n\n\n\n## Related links\n * CompressAI Library: https://github.com/InterDigitalInc/CompressAI\n","funding_links":[],"readme_doi_urls":[],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://ost.ecosyste.ms/api/v1/projects/301428","html_url":"https://ost.ecosyste.ms/projects/301428"}