{"id":325194,"name":"reformatters","description":"Reformat weather datasets into Zarr.","url":"https://github.com/dynamical-org/reformatters","last_synced_at":"2026-08-13T13:35:58.095Z","repository":{"id":332212492,"uuid":"859043226","full_name":"dynamical-org/reformatters","owner":"dynamical-org","description":"Reformat weather datasets into Zarr","archived":false,"fork":false,"pushed_at":"2026-08-07T16:25:23.000Z","size":147086,"stargazers_count":38,"open_issues_count":37,"forks_count":9,"subscribers_count":2,"default_branch":"main","last_synced_at":"2026-08-07T16:59:24.797Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"https://dynamical.org/catalog","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"bsd-3-clause","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/dynamical-org.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":"CITATION.cff","codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":".zenodo.json","notice":null,"maintainers":null,"copyright":null,"agents":"AGENTS.md","dco":null,"cla":null,"disclosure":null}},"created_at":"2024-09-18T01:28:17.000Z","updated_at":"2026-08-07T04:40:09.000Z","dependencies_parsed_at":null,"dependency_job_id":"5e3f56e7-5ce9-4d83-9123-43ce18ca8fa0","html_url":"https://github.com/dynamical-org/reformatters","commit_stats":null,"previous_names":["dynamical-org/reformatters"],"tags_count":1,"template":false,"template_full_name":null,"purl":"pkg:github/dynamical-org/reformatters","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dynamical-org","download_url":"https://codeload.github.com/dynamical-org/reformatters/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":36387897,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-08-06T04:43:03.162Z","status":"online","status_checked_at":"2026-08-07T02:00:06.708Z","response_time":57,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"dynamical-org","name":"dynamical-org","uuid":"156974863","kind":"organization","description":null,"email":null,"website":null,"location":null,"twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/156974863?v=4","repositories_count":1,"last_synced_at":"2024-01-20T21:33:46.041Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/dynamical-org","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2024-01-20T21:33:46.044Z","updated_at":"2024-01-20T21:33:46.044Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dynamical-org","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dynamical-org/repositories"},"packages":[],"commits":{"id":11311122,"full_name":"dynamical-org/reformatters","default_branch":"master","total_commits":877,"total_committers":11,"total_bot_commits":17,"total_bot_committers":1,"mean_commits":79.72727272727273,"dds":0.3158494868871151,"past_year_total_commits":500,"past_year_total_committers":9,"past_year_total_bot_commits":17,"past_year_total_bot_committers":1,"past_year_mean_commits":55.55555555555556,"past_year_dds":0.27,"last_synced_at":"2026-08-12T13:06:08.673Z","last_synced_commit":"acdb1990c98b4c4a01f40a416eecdf3afa12113d","created_at":"2025-09-23T00:26:10.247Z","updated_at":"2026-08-12T13:06:07.046Z","committers":[{"name":"Alden Keefe Sampson","email":"alden@dynamical.org","login":"aldenks","count":600},{"name":"Alex","email":"mosegontar","login":"mosegontar","count":71},{"name":"Sam Neubardt","email":"sam@upstream.tech","login":"samn","count":56},{"name":"Margo Crawford","email":"margo@upstream.tech","login":"margocrawf","count":42},{"name":"Marshall","email":"marshall@dynamical.org","login":"mrshll","count":37},{"name":"Jack Kelly","email":"jack@OpenClimateFix.org","login":"JackKelly","count":20},{"name":"dependabot[bot]","email":"49699333+dependabot[bot]","login":"dependabot[bot]","count":17},{"name":"Dan Katz","email":"danielkatz125@gmail.com","login":"dan-katz","count":11},{"name":"Tony Cannistra","email":"tony.cannistra@gmail.com","login":"acannistra","count":8},{"name":"Copilot","email":"198982749+Copilot","login":"Copilot","count":8},{"name":"Lauren Gulland","email":"lauren@upstream.tech","login":"laurengulland","count":7}],"past_year_committers":[{"name":"Alden Keefe Sampson","email":"alden@dynamical.org","login":"aldenks","count":365},{"name":"Marshall","email":"marshall@dynamical.org","login":"mrshll","count":37},{"name":"Alex","email":"mosegontar","login":"mosegontar","count":36},{"name":"dependabot[bot]","email":"49699333+dependabot[bot]","login":"dependabot[bot]","count":17},{"name":"Jack Kelly","email":"jack@OpenClimateFix.org","login":"JackKelly","count":17},{"name":"Sam Neubardt","email":"sam@upstream.tech","login":"samn","count":13},{"name":"Copilot","email":"198982749+Copilot","login":"Copilot","count":8},{"name":"Lauren Gulland","email":"lauren@upstream.tech","login":"laurengulland","count":6},{"name":"Tony Cannistra","email":"tony.cannistra@gmail.com","login":"acannistra","count":1}],"commits_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/commits","host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-08-13T00:00:06.963Z","repositories_count":6649999,"commits_count":933997870,"contributors_count":39760896,"owners_count":1336013,"committers_count":3301464,"icon_url":"https://github.com/github.png","host_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories"}},"issues_stats":{"full_name":"dynamical-org/reformatters","html_url":"https://github.com/dynamical-org/reformatters","last_synced_at":"2026-08-12T13:06:11.660Z","status":null,"issues_count":50,"pull_requests_count":324,"avg_time_to_close_issue":2915660.2,"avg_time_to_close_pull_request":160116.18532818533,"issues_closed_count":20,"pull_requests_closed_count":259,"pull_request_authors_count":10,"issue_authors_count":5,"avg_comments_per_issue":3.06,"avg_comments_per_pull_request":0.3117283950617284,"merged_pull_requests_count":230,"bot_issues_count":0,"bot_pull_requests_count":16,"past_year_issues_count":46,"past_year_pull_requests_count":159,"past_year_avg_time_to_close_issue":1683176.4736842106,"past_year_avg_time_to_close_pull_request":203226.6567164179,"past_year_issues_closed_count":19,"past_year_pull_requests_closed_count":134,"past_year_pull_request_authors_count":7,"past_year_issue_authors_count":4,"past_year_avg_comments_per_issue":2.5869565217391304,"past_year_avg_comments_per_pull_request":0.3522012578616352,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":16,"past_year_merged_pull_requests_count":108,"created_at":"2025-07-16T12:49:38.101Z","updated_at":"2026-08-12T13:06:11.660Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/dynamical-org%2Freformatters/issues","issue_labels_count":{"enhancement":14,"bug":1},"pull_request_labels_count":{"dependencies":16,"python:uv":14,"github_actions":2,"codex":1},"issue_author_associations_count":{"MEMBER":44,"COLLABORATOR":4,"NONE":1,"CONTRIBUTOR":1},"pull_request_author_associations_count":{"MEMBER":203,"CONTRIBUTOR":114,"NONE":6,"COLLABORATOR":1},"issue_authors":{"mrshll":29,"aldenks":14,"JackKelly":5,"briannapagan":1,"samn":1},"pull_request_authors":{"aldenks":150,"mrshll":49,"mosegontar":43,"samn":36,"dependabot[bot]":16,"dan-katz":10,"margocrawf":9,"acannistra":7,"JackKelly":3,"mayadebellis":1},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2026-08-13T00:00:06.201Z","repositories_count":16028902,"issues_count":40463188,"pull_requests_count":130880990,"authors_count":11443198,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{"enhancement":13,"bug":1},"past_year_pull_request_labels_count":{"dependencies":16,"python:uv":14,"github_actions":2,"codex":1},"past_year_issue_author_associations_count":{"MEMBER":43,"COLLABORATOR":3},"past_year_pull_request_author_associations_count":{"MEMBER":120,"CONTRIBUTOR":38,"COLLABORATOR":1},"past_year_issue_authors":{"mrshll":29,"aldenks":13,"JackKelly":3,"samn":1},"past_year_pull_request_authors":{"aldenks":67,"mrshll":47,"mosegontar":19,"dependabot[bot]":16,"samn":8,"JackKelly":1,"mayadebellis":1},"maintainers":[{"login":"aldenks","count":164,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/aldenks"},{"login":"mrshll","count":76,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mrshll"},{"login":"samn","count":7,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/samn"},{"login":"JackKelly","count":4,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/JackKelly"},{"login":"mayadebellis","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mayadebellis"}],"active_maintainers":[{"login":"aldenks","count":80,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/aldenks"},{"login":"mrshll","count":76,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mrshll"},{"login":"samn","count":7,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/samn"},{"login":"JackKelly","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/JackKelly"},{"login":"mayadebellis","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mayadebellis"}]},"events":{"total":{"DeleteEvent":332,"PullRequestEvent":522,"ForkEvent":5,"IssuesEvent":31,"WatchEvent":20,"IssueCommentEvent":197,"PushEvent":1089,"PullRequestReviewCommentEvent":421,"PullRequestReviewEvent":428,"CreateEvent":350},"last_year":{"DeleteEvent":197,"PullRequestEvent":222,"IssuesEvent":28,"WatchEvent":12,"IssueCommentEvent":96,"PushEvent":611,"PullRequestReviewEvent":160,"PullRequestReviewCommentEvent":192,"CreateEvent":199}},"keywords":[],"dependencies":[],"score":6.715383386334681,"created_at":"2025-09-23T00:25:39.587Z","updated_at":"2026-08-13T13:35:58.097Z","avatar_url":"https://github.com/dynamical-org.png","language":"Python","category":"Atmosphere","sub_category":"Meteorological Observation and Forecast","monthly_downloads":0,"total_dependent_repos":0,"total_dependent_packages":0,"readme":"# dynamical.org reformatters\n\nReformat weather datasets into zarr.\n\nBrowse the datasets produced by this repo at https://dynamical.org/catalog/.\n\n* See [AGENTS.md](AGENTS.md) for an overview of the approach and this repository.\n* [Develop a new dataset](docs/dataset_development_guide.md) end to end, or dive into just the [implementation](docs/implementation_guide.md).\n* [Add a new variable](docs/add_new_variable.md) to an existing dataset.\n\n[![DOI](https://zenodo.org/badge/859043226.svg)](https://doi.org/10.5281/zenodo.18777399)\n\n## Local development\n\nWe use\n* `uv` to manage dependencies and python environments\n* `ruff` for linting and formatting\n* `ty` for type checking\n* `pytest` for testing\n* `prek` to automatically lint and format as you git commit\n\n### Setup\n1. [Install uv](https://docs.astral.sh/uv/getting-started/installation/)\n1. Run `uv run prek install` to setup the git hooks\n1. If you use VSCode, you may want to install the extensions (ruff) it will recommend when you open this folder\n\n### Running locally\n\n* `uv run main --help` - list all datasets\n* `uv run main \u003cDATASET_ID\u003e update-template`\n* `uv run main \u003cDATASET_ID\u003e backfill-local \u003cAPPEND_DIM_END\u003e`\n\n### Development commands\n* Add dependency: `uv add \u003cpackage\u003e [--dev]`. Use `--dev` to add a development only dependency.\n* Lint: `uv run ruff check [--fix]`\n* Type check: `uv run ty check`\n* Format: `uv run ruff format`\n* Tests: \n   * Run tests in parallel on all available cores: `uv run pytest`\n   * Run tests serially: `uv run pytest -n 0`\n\n## Deploying to the cloud\n\nTo reformat a large archive we parallelize work across multiple cloud servers.\n\nWe use\n* `docker` to package the code and dependencies\n* `kubernetes` indexed jobs to run work in parallel\n\n### Setup\n\n1. Install `docker` and `kubectl`. Make sure `docker` can be found at `/usr/bin/docker` and `kubectl` at `/usr/bin/kubectl`.\n1. Setup a docker image repository and export the `DOCKER_REPOSITORY` environment variable in your local shell. e.g. `export DOCKER_REPOSITORY=container.registry/\u003cproject-id\u003e/reformatters/main`. Follow your registry's instructions to allow your docker to authenticate and push images to the registry.\n1. Setup a kubernetes cluster and configure kubectl to point to your cluster. e.g. `aws eks update-kubeconfig --region \u003cregion\u003e --name \u003ccluster-name\u003e`, `gcloud container clusters get-credentials \u003ccluster-name\u003e --region \u003cregion\u003e --project \u003cproject\u003e`, etc.\n1. Create a kubectl secret containing a single json encoded value to be passed to fsspec `storage_options` or splatted as keyword arguments to an icechunk storage opener `kubectl create secret generic your-destination-storage-options-key --from-literal=contents='{\"key\": \"...\", \"secret\": \"...\"}'`. See `storage.py`.\n","funding_links":[],"readme_doi_urls":["https://doi.org/10.5281/zenodo.18777399"],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://ost.ecosyste.ms/api/v1/projects/325194","html_url":"https://ost.ecosyste.ms/projects/325194"}