diff --git a/healthdata_influx/.dockerignore b/healthdata_influx/.dockerignore new file mode 100644 index 0000000..cd86fa4 --- /dev/null +++ b/healthdata_influx/.dockerignore @@ -0,0 +1,2 @@ +data +docker-compose diff --git a/healthdata_influx/.gitignore b/healthdata_influx/.gitignore new file mode 100644 index 0000000..0053ac5 --- /dev/null +++ b/healthdata_influx/.gitignore @@ -0,0 +1,96 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +data/ +.DS_Store +config.yml +!docker-compose/config.yml + +.vscode diff --git a/healthdata_influx/Dockerfile b/healthdata_influx/Dockerfile new file mode 100644 index 0000000..1af93bd --- /dev/null +++ b/healthdata_influx/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3 + +WORKDIR /app + +COPY requirements.txt /app +RUN pip install --no-cache-dir -r requirements.txt + +COPY healthdata_influx /app + +CMD [ "python3", "import.py", "/data/export.xml" ] \ No newline at end of file diff --git a/healthdata_influx/README.md b/healthdata_influx/README.md new file mode 100644 index 0000000..f6dfada --- /dev/null +++ b/healthdata_influx/README.md @@ -0,0 +1,90 @@ +# healthdata_influx +Imports Apple Health Data into InfluxDB. + +![Grafana Screenshot](https://www.tannr.com/wp-content/uploads/2017/03/grafana.png "Grafana Screenshot") +Visualizing InfluxDB using [Grafana](https://grafana.com/). + +## How to export iOS Health Data +1. Go to the Health App +2. Tap the profile image at the top right +3. Tap "Export Health Data" +4. Save the `export.zip` file and extract its XML contents (`export.xml`) somewhere accessible by this script. + +## Running as a complete service with Docker Compose (bonus Grafana Graphs!) + +This is the easiest way to get up and running quickly. This will spin up the importer, an InfluxDB database, and Grafana with a default dashboard ready to go. + +#### Requirements: + +* [Docker](https://www.docker.com/) with [Docker Compose](https://docs.docker.com/compose/) + +#### Installation: + +* Create a `data` directory at the project root and add the `export.xml` inside it + +#### Building: + +`docker-compose build` + +#### Running: + +1. `docker-compose up` (add `-d` to run in daemon mode) +2. Access Grafana in your web browser: [http://localhost:3000](http://localhost:3000) + +Username: `admin` Password: `admin` + +#### Refreshing data: + +1. Replace `data/export.xml` with a new version +2. `docker-compose run importer` + +## Running as a Python module or stand-alone script. + +#### Requirements: + +* [Python 3](https://www.python.org/) +* An accessible [InfluxDB](https://www.influxdata.com/) instance + +#### Installation: + +* `pip install -r requirements.txt` +* Rename `config_sample.yml` to `config.yml` + +#### Configuration: + +* Edit `config.yml` to match your InfluxDB settings (host, auth, etc.) + +#### Usage: + +1. Export Health Data from iOS device +2. `python3 import.py export.xml` + + +#### See also: +`python import.py --help` + +## Running as a stand-alone Docker container + +#### Requirements: + +* [Docker](https://www.docker.com/) + +#### Installation: + +* Create a `data` directory at the repo root and add the `export.xml` inside it. + * (note that this can be anywhere if the volume mount point on the `docker run` command is changed) + +#### Configuration: + +* Edit `config.yml` to match your InfluxDB settings (host, auth, etc.) + +#### Building: + +`docker build . -t twstokes/healthdata_influx` + +#### Running (at the repo root): + +`docker run -v $PWD/data:/data:ro -v $PWD/config.yml:/app/config.yml:ro twstokes/healthdata_influx` + +## Todo / Notes: +* Does not support "Mindful Sessions" diff --git a/healthdata_influx/docker-compose.yml b/healthdata_influx/docker-compose.yml new file mode 100644 index 0000000..a07d65d --- /dev/null +++ b/healthdata_influx/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' + +services: + importer: + build: . + volumes: + - ./data:/data:ro + - ./docker-compose/config.yml:/app/config.yml:ro diff --git a/healthdata_influx/docker-compose/config.yml b/healthdata_influx/docker-compose/config.yml new file mode 100644 index 0000000..86d10d8 --- /dev/null +++ b/healthdata_influx/docker-compose/config.yml @@ -0,0 +1,12 @@ +# configure with params from the docs: +# http://influxdb-python.readthedocs.io/en/latest/api-documentation.html#influxdbclient + +influxdb: + client: + host: home.toozhao.com + port: 28086 + database: home + username: junv + password: "@wahyd4" + write_points: + batch_size: 1000 diff --git a/healthdata_influx/docker-compose/env.grafana b/healthdata_influx/docker-compose/env.grafana new file mode 100644 index 0000000..e69de29 diff --git a/healthdata_influx/docker-compose/env.influxdb b/healthdata_influx/docker-compose/env.influxdb new file mode 100644 index 0000000..e69de29 diff --git a/healthdata_influx/docker-compose/grafana_dashboard_providers/default.yml b/healthdata_influx/docker-compose/grafana_dashboard_providers/default.yml new file mode 100644 index 0000000..ba9f89f --- /dev/null +++ b/healthdata_influx/docker-compose/grafana_dashboard_providers/default.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: +- name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/healthdata_influx/docker-compose/grafana_dashboards/default.json b/healthdata_influx/docker-compose/grafana_dashboards/default.json new file mode 100644 index 0000000..15d5786 --- /dev/null +++ b/healthdata_influx/docker-compose/grafana_dashboards/default.json @@ -0,0 +1,176 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "$$hashKey": "object:172", + "groupBy": [ + { + "params": [ + "24h" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "HKQuantityTypeIdentifierStepCount", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Steps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-2y", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Health Data", + "uid": "k0lVpINmz", + "version": 1 +} \ No newline at end of file diff --git a/healthdata_influx/docker-compose/grafana_datasources/grafana_influx.yaml b/healthdata_influx/docker-compose/grafana_datasources/grafana_influx.yaml new file mode 100644 index 0000000..427d851 --- /dev/null +++ b/healthdata_influx/docker-compose/grafana_datasources/grafana_influx.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: InfluxDB + type: influxdb + access: proxy + database: health + user: grafana + password: grafana + url: http://influxdb:8086 \ No newline at end of file diff --git a/healthdata_influx/healthdata_influx/db.py b/healthdata_influx/healthdata_influx/db.py new file mode 100644 index 0000000..c60c8d8 --- /dev/null +++ b/healthdata_influx/healthdata_influx/db.py @@ -0,0 +1,65 @@ +"""Loads a configuration and imports data points into InfluxDB""" +from datetime import datetime, timezone +import yaml +from influxdb import InfluxDBClient + +class InfluxDBUploader: + """Uploads data points to an InfluxDB instance""" + def __init__(self, config_path): + self.config = self._load_config(config_path) + + def upload(self, data_points=None): + """Uploads data points to InfluxDB""" + if data_points is not None and len(data_points): + client = InfluxDBClient(**self.config['influxdb']['client']) + # only creates a DB if none exists + client.create_database(self.config['influxdb']['client']['database']) + client.write_points(points=data_points, **self.config['influxdb']['write_points']) + + def create_point(self, measurement, time, fields, tags=None): + """Helps enforce proper InfluxDB point creation""" + # tags can be an empty dict + if tags is None: + tags = {} + + if not isinstance(measurement, str): + raise TypeError('Measurement must be a string.') + + if not isinstance(time, datetime): + raise TypeError('Time must be a datetime object.') + + if not isinstance(fields, dict): + raise TypeError('Fields must be a dictionary.') + elif len(fields) < 1: + # there must be at least one field + raise ValueError('Fields must contain at least one field.') + + if not isinstance(tags, dict): + raise TypeError('Tags must be a dictionary.') + + # convert datetime object to string + time = self._create_influx_time(time) + + point = { + 'tags': tags, + 'time': time, + 'fields': fields, + 'measurement': measurement + } + + return point + + def _load_config(self, config_path): + """Loads config for this script""" + with open(config_path) as file: + config = yaml.safe_load(file) + return config + + def _create_influx_time(self, time): + """ + Takes in a datetime object + Returns a datetime string InfluxDB expects, in UTC + """ + # save as correct format in UTC timezone + converted_time = time.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + return converted_time diff --git a/healthdata_influx/healthdata_influx/import.py b/healthdata_influx/healthdata_influx/import.py new file mode 100644 index 0000000..4a87b3d --- /dev/null +++ b/healthdata_influx/healthdata_influx/import.py @@ -0,0 +1,154 @@ +""" +Parses an Apple Health export file +and imports into InfluxDB +""" +import sys +import argparse +from datetime import datetime +from lxml import etree +from db import InfluxDBUploader + +class Importer: + """ + Importer parses an Apple Health XML file + and uploads Records to a database + """ + def __init__(self, uploader, dry=False, buffer_size=50000): + if dry: + print('Dry run - no database changes will be made.') + + self.dry = dry + self.uploader = uploader + # how many records to Buffer before flushing to the database + # adjusting this will affect memory consumption + self.buffer_size = buffer_size + + def upload(self, points): + """ + Sends points to the uploader if not a dry run. + """ + if not self.dry: + self.uploader.upload(points) + + def parse_and_upload(self, export_path): + """ + Takes InfluxDB configuration and Apple Health Data file paths + Uploads to InfluxDB + """ + + def create_flusher(buffer, size): + def flusher(records): + print("Flushing {} points to DB. Current total: {}".format(size, records)) + self.upload(buffer[:size]) + # clean up + del buffer[:size] + return flusher + + try: + print('Opening export file...') + with open(export_path, mode='rb') as file: + context = self.get_record_iterator(file) + + point_buffer = [] + total_records, success_records = (0, 0) + flusher = create_flusher(point_buffer, self.buffer_size) + + for idx, (_, record) in enumerate(context): + total_records += 1 + + try: + point = self.mung_record_to_point(record) + point_buffer.append(point) + success_records += 1 + except Exception as error: + output_mung_error(error, record, idx+1) + + if len(point_buffer) > self.buffer_size - 1: + flusher(total_records) + + # memory cleanup + record.clear() + while record.getprevious() is not None: + del record.getparent()[0] + + # upload the rest + self.upload(point_buffer) + + print("Successful uploads: {}".format(success_records)) + print("Total records: {}".format(total_records)) + except Exception as error: + print('Failure!') + print(sys.exc_info()) + print(error) + + def mung_record_to_point(self, record): + """ + Returns an InfluxDB point for a health record XML element + """ + attr = record.attrib + + if ('endDate' not in attr + or 'value' not in attr + or 'type' not in attr): + raise ValueError('Failed to find all required fields.') + + tags = {} + fields = {} + + value = attr['value'] + end_date = attr['endDate'] + measurement = attr['type'] + + try: + # try to convert to a number + value = float(value) + except ValueError: + # carry on as a string + pass + + # set the fields + fields['value'] = value + # convert to datetime obj + time = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S %z') + + if 'unit' in attr: + tags['unit'] = attr['unit'] + if 'sourceName' in attr: + tags['source'] = attr['sourceName'] + + point = self.uploader.create_point(measurement, time, fields, tags) + + return point + + def get_record_iterator(self, file): + """ + Takes in an Apple Health Data + export file, returns iterator for Record elements + """ + return etree.iterparse(file, events=('end',), tag='Record') + +def output_mung_error(error, record, index): + print("Couldn't convert record to point:", error) + print(etree.tostring(record)) + print("Record index: {}".format(index)) + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser(description='Imports Apple Health Data to InfluxDB') + + PARSER.add_argument('--config_path', help='InfluxDB config file path', default='./config.yml') + PARSER.add_argument('--dry', help='Dry run - no DB changes', action='store_true', default=False) + PARSER.add_argument('export_path', help='Apple Health Data export file path') + + ARGS = PARSER.parse_args() + + try: + UPLOADER = InfluxDBUploader(ARGS.config_path) + print('InfluxDB uploader loaded.') + IMPORTER = Importer(UPLOADER, ARGS.dry) + print('Importer loaded.') + IMPORTER.parse_and_upload(ARGS.export_path) + except FileNotFoundError: + print('Could not load InfluxDB configuration file!') + except Exception as error: + print('Failed to initialize InfluxDB!') + print(error) diff --git a/healthdata_influx/requirements.txt b/healthdata_influx/requirements.txt new file mode 100644 index 0000000..493509e --- /dev/null +++ b/healthdata_influx/requirements.txt @@ -0,0 +1,3 @@ +lxml +pyyaml +influxdb