Commit 912a3564 authored by Mattias Jakobsson's avatar Mattias Jakobsson
Browse files

Merge branch 'release/0.7'

parents 06ccefe4 099e2b87
Pipeline #239202711 passed with stages
in 4 minutes and 4 seconds
[flake8]
ignore = E203
max-line-length = 88
/build/
/.coverage
/dist/
/pupygrib.egg-info/
/src/pupygrib.egg-info/
/.pytest_cache/
/.tox/
__pycache__/
stages:
- test
- build
- test
- publish
image: python:3.6
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/cache/pip"
cache:
key: py36
paths:
- cache
before_script:
- python -V
- pip install virtualenv
- virtualenv cache/venv
- python -m venv cache/venv
- source cache/venv/bin/activate
- pip install --upgrade pip
- pip install -r requirements/ci.txt
test:coding-style:
script:
- tox -e check-manifest,black,flake8
test:py36:
variables:
TOX_SKIP_ENV: "(?!py36).*"
script:
- tox
.py36:
image: python:3.6
cache:
key: py36
paths:
- cache
test:py37:
.py37:
image: python:3.7
cache:
key: py37
paths:
- cache
variables:
TOX_SKIP_ENV: "(?!py37).*"
script:
- tox
test:py38:
.py38:
image: python:3.8
cache:
key: py38
paths:
- cache
variables:
TOX_SKIP_ENV: "(?!py38).*"
script:
- tox
test:coverage:
script:
- tox -e coverage
.py39:
image: python:3.9
cache:
key: py39
paths:
- cache
build:
stage: build
extends: .py39
script:
- python setup.py sdist bdist_wheel
artifacts:
paths:
- dist/pupygrib*.whl
- dist/pupygrib*.tar.gz
only:
- tags
- dist/*
.codetest:
stage: test
extends: .py39
needs: []
test:format:
extends: .codetest
script:
- nox -s format
test:imports:
extends: .codetest
script:
- nox -s imports
test:lint:
extends: .codetest
script:
- nox -s lint
test:manifest:
extends: .codetest
script:
- nox -s manifest
test:doctest:
extends: .codetest
script:
- nox -s doctest
test:coverage:
extends: .codetest
script:
- nox -s coverage
.typing:
stage: test
needs: ["build"]
script:
- nox -s typing
.unit:
stage: test
needs: ["build"]
script:
- nox -s unittest
artifacts:
reports:
junit: tests_py3*.xml
test:typing-py36:
extends: [".py36", ".typing"]
test:unit-py36:
extends: [".py36", ".unit"]
test:typing-py37:
extends: [".py37", ".typing"]
test:unit-py37:
extends: [".py37", ".unit"]
test:typing-py38:
extends: [".py38", ".typing"]
test:unit-py38:
extends: [".py38", ".unit"]
test:typing-py39:
extends: [".py39", ".typing"]
test:unit-py39:
extends: [".py39", ".unit"]
test:dists:
stage: test
extends: .py39
needs: ["build"]
script:
- twine check --strict dist/*
.publish:
stage: publish
extends: .py39
needs: ["build"]
script:
- twine upload dist/*
......
repos:
- repo: https://github.com/ambv/black
rev: 19.3b0
rev: "20.8b1"
hooks:
- id: black
- repo: https://github.com/mgedmin/check-manifest
rev: "0.45"
hooks:
- id: check-manifest
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.2.3
rev: "v3.3.0"
hooks:
- id: check-added-large-files
- id: check-docstring-first
......@@ -16,11 +20,14 @@ repos:
- id: end-of-file-fixer
- id: fix-encoding-pragma
args: [--remove]
- id: flake8
- id: mixed-line-ending
args: [--fix, lf]
- id: trailing-whitespace
- repo: https://notabug.org/mjakob/pre-commit-check-manifest
rev: v0.2.0
- repo: https://github.com/pycqa/flake8
rev: "3.8.4"
hooks:
- id: check-manifest
- id: flake8
- repo: https://github.com/pycqa/isort
rev: "5.6.4"
hooks:
- id: isort
# Change log
## 0.7.0
* Add type annotations to the source code.
* Revise the Message classes to better support typing. The sections
of a GRIB message are now rather accessed by name than by index.
* Add a `get_time()` method to the message classes.
* Add official support for Python 3.9.
## 0.6.1
* Fix unpacking messages with an odd number of 12-bit values.
......
include CHANGES.md
include .gitlab-ci.yml
include .flake8
include LICENSE.txt
include mypy.ini
include noxfile.py
include .pre-commit-config.yaml
include pyproject.toml
include requirements/*.in
include requirements/*.txt
recursive-include tests *.py
recursive-include tests/data *
include tox.ini
......@@ -31,8 +31,10 @@ The planned features are:
## Requirements
* [Python](https://www.python.org) 3.6 or later (3.6-3.8 are tested).
* [Numpy](http://www.numpy.org) (1.12-1.19 are tested).
* [Python](https://www.python.org) 3.6 or later (3.6-3.9 are tested).
3.8 or later is recommended for the better typing support.
* [Numpy](http://www.numpy.org) 1.12-1.19 are tested with compatible
Python versions.
## Installation
......@@ -52,18 +54,19 @@ and
[edition 2](http://apps.ecmwf.int/codes/grib/format/grib2/overview)
that can be used as references.
Iterate over the messages in a GRIB file and extract the coordinates
and values:
Iterate over the messages in a GRIB file and extract the time,
coordinates and values:
``` python
>>> import pupygrib
>>> with open('tests/data/regular_latlon_surface.grib1', 'rb') as stream:
... for i, msg in enumerate(pupygrib.read(stream), 1):
... lons, lats = msg.get_coordinates()
... time = msg.get_time()
... values = msg.get_values()
... print("Message {}: {:.3f} {}".format(i, values.mean(), lons.shape))
... print("Message {}: {} {:.3f} {}".format(i, time, values.mean(), lons.shape))
...
Message 1: 291.585 (31, 16)
Message 1: 2008-02-06 12:00:00 291.585 (31, 16)
```
......@@ -72,11 +75,11 @@ Access a section of a GRIB message and print its fields:
``` python
>>> with open('tests/data/regular_latlon_surface.grib1', 'rb') as stream:
... msg, = pupygrib.read(stream)
>>> sorted(msg[0].fieldnames) # fieldnames is a set, so we can't trust the order
>>> sorted(msg.is_.fieldnames) # fieldnames is a set, so we can't trust the order
['editionNumber', 'identifier', 'totalLength']
>>> msg[0].totalLength
1100
>>> msg[3] is None # the bit-map section is not included in this message
>>> msg.bitmap is None # the bit-map section is not included in this message
True
```
......@@ -87,18 +90,20 @@ True
Pull requests (against the `develop` branch) are most welcome! I do
ask that you add test cases and update the documentation (this README
for now) for any new features. Run the code through the
auto-formatter [black](https://black.readthedocs.io/en/stable/) and
make sure that all checks (coding style, unit tests, and the
auto-formatters [black][] and [isort][] and make sure that all checks
(coding style, static type check, unit tests, and the
[manifest](MANIFEST.in)) pass without any warnings or errors. The
easiest way to do this is to install the requirements in
[requirements/dev.txt](requirements/dev.txt) and run [tox][]:
[requirements/dev.txt](requirements/dev.txt) and run [nox][]:
```console
$ pip install -r requirements/dev.txt
$ tox
$ nox
```
[tox]: http://tox.readthedocs.org/
[black]: https://black.readthedocs.io/en/stable/
[isort]: https://pycqa.github.io/isort/
[nox]: https://nox.thea.codes/
## License
......
[mypy]
warn_redundant_casts = true
warn_unused_configs = true
check_untyped_defs = true
disallow_any_generics = true
disallow_incomplete_defs = true
disallow_subclassing_any = true
disallow_untyped_calls = true
disallow_untyped_defs = true
disallow_untyped_decorators = true
no_implicit_optional = true
no_implicit_reexport = true
strict_equality = true
warn_unused_ignores = true
[mypy-noxfile]
disallow_untyped_calls = false
disallow_untyped_defs = false
[mypy-numpy.*]
ignore_missing_imports = true
[mypy-setuptools]
ignore_missing_imports = true
"""Nox configuration for pupygrib."""
import glob
import os
import sys
import nox
from nox.command import CommandFailed
nox.options.sessions = [
"format",
"imports",
"lint",
"manifest",
"typing",
"doctest",
"unittest",
"coverage",
]
python_sources = ["noxfile.py", "src/pupygrib", "setup.py", "tests"]
numpy_versions = [
"1.12.1",
"1.13.3",
"1.14.6",
"1.15.4",
"1.16.6",
"1.17.5",
"1.18.5",
"1.19.4",
]
@nox.session
def requirements(session):
"""Compile all requirement files"""
session.install("pip-tools")
def pipcompile(outputfile, *inputfiles):
session.run(
"pip-compile",
"--quiet",
"--allow-unsafe",
"--output-file",
outputfile,
*inputfiles,
*session.posargs,
env={"CUSTOM_COMPILE_COMMAND": "nox -s requirements"},
)
pipcompile("requirements/test.txt", "requirements/test.in")
pipcompile("requirements/dev.txt", "requirements/dev.in")
pipcompile("requirements/ci.txt", "requirements/ci.in")
@nox.session
def format(session):
"""Check the source code format with Black"""
session.install("black ~= 20.8b1")
session.run("black", "--check", "--quiet", *python_sources)
@nox.session
def imports(session):
"""Check the source code imports with isort"""
session.install("isort ~= 5.6.4")
session.run("isort", "--check", *python_sources)
@nox.session
def lint(session):
"""Check the source code with flake8"""
session.install("flake8 ~= 3.8.4")
session.run("flake8", *python_sources)
@nox.session
def manifest(session):
"""Check that the MANIFEST.in is up-to-date"""
session.install("check-manifest ~= 0.45")
session.run("check-manifest")
@nox.session
def typing(session):
"""Run a static type check with mypy"""
session.install("mypy == 0.790")
session.install("-r", "requirements/test.txt")
session.install("-e", ".")
session.run("mypy", "src/pupygrib", "tests")
@nox.session
def doctest(session):
"""Check the code examples in the documentation"""
session.install("-r", "requirements/test.txt")
session.install(f"numpy ~= {numpy_versions[-1]}")
session.install("-e", ".")
session.run("pytest", "--doctest-glob=*.md", *glob.glob("*.md"))
@nox.session
def coverage(session):
"""Check unit test coverage"""
session.install("-r", "requirements/test.txt")
session.install(f"numpy ~= {numpy_versions[-1]}")
session.install("-e", ".")
session.run("pytest", "--cov=pupygrib", "tests", *session.posargs)
def get_junit_prefix(numpy):
pyversion = f"py{sys.version_info.major}{sys.version_info.minor}"
npversion = f"np{''.join(numpy.split('.')[:2])}"
return f"tests_{pyversion}_{npversion}"
@nox.session
@nox.parametrize("numpy", numpy_versions)
def unittest(session, numpy):
"""Run the unit tests"""
session.install("-r", "requirements/test.txt")
try:
session.install("--only-binary", "numpy", f"numpy ~= {numpy}")
except CommandFailed:
session.skip("No numpy wheel for this python version")
if "CI" in os.environ:
session.install("--find-links", "dist", "--no-deps", "--pre", "pupygrib")
junitprefix = get_junit_prefix(numpy)
ciargs = ["--junit-xml", f"{junitprefix}.xml", "--junit-prefix", junitprefix]
else:
session.install("-e", ".")
ciargs = []
session.run("pytest", "tests", *ciargs, *session.posargs)
"""An edition 1 GRIB message."""
import numpy
from pupygrib import base
from pupygrib import binary
from pupygrib import fields as basefields # Python gets confused without alias
from pupygrib.edition1 import section1
from pupygrib.edition1 import section2
from pupygrib.edition1 import section3
from pupygrib.edition1 import section4
class IndicatorSection(base.Section):
"""The indicator section (0) of an edition 1 GRIB message."""
identifier = basefields.BytesField(1, 4)
totalLength = basefields.Uint24Field(5)
editionNumber = basefields.Uint8Field(8)
class EndSection(base.Section):
"""The end section (5) of an edition 1 GRIB message."""
endOfMessage = basefields.BytesField(1, 4)
class Message(base.Message):
"""An edition 1 GRIB message."""
def get_coordinates(self):
"""Return the coordinates of this message's data points."""
griddesc = self[2]
if griddesc is None:
raise NotImplementedError("pupygrib does not support catalogued grids")
return griddesc._get_coordinates()
def get_values(self):
"""Return the data values of this message."""
values = self[4]._unpack_values()
values = self[1]._scale_values(values)
bitmapdesc = self[3]
if bitmapdesc:
mask = ~numpy.array(bitmapdesc.bitmap, dtype=bool)
mvalues = numpy.empty(len(mask), dtype=float)
mvalues[~mask] = values
values = numpy.ma.array(mvalues, mask=mask)
griddesc = self[2]
if griddesc:
values = griddesc._order_values(values)
return values
def _get_section0(self):
return IndicatorSection(self._data, 0, 8)
def _get_section1(self):
offset = self[0].end
length = binary.unpack_uint24_from(self._data, offset)
return section1.get_section(self._data, offset, length)
def _get_section2(self):
proddef = self[1]
if not proddef.section1Flags & 0x80:
return None
offset = proddef.end
length = binary.unpack_uint24_from(self._data, offset)
return section2.get_section(self._data, offset, length)
def _get_section3(self):
proddef = self[1]
if not proddef.section1Flags & 0x40:
return None
offset = (self[2] or proddef).end
length = binary.unpack_uint24_from(self._data, offset)
return section3.BitMapSection(self._data, offset, length)
def _get_section4(self):
offset = (self[3] or self[2] or self[1]).end
length = binary.unpack_uint24_from(self._data, offset)
return section4.get_section(self._data, offset, length)
def _get_section5(self):
return EndSection(self._data, self[4].end, 4)
"""Fields for edition 1 GRIB messages."""
from pupygrib import base
from pupygrib import binary
class FloatField(base.Field):
"""A 32-bit GRIB 1 floating point field."""
def get_value(self, section, offset):
return binary.unpack_grib1float_from(section._data, offset)
[build-system]
requires = ["setuptools", "wheel"]
[tool.black]
target_version = ['py36', 'py37']
exclude = '''
/(
\.direnv
| \.git
| \.pytest_cache
| \.tox
| \.venv
| build
| cache
| dist
/)
'''
[tool.check-manifest]
ignore = [".gitlab-ci.yml"]
[tool.coverage.run]
source = ["pupygrib"]