Add docker-compose to import ios health data into inflxudb

healthdata_influx/.dockerignore

@@ -0,0 +1,2 @@
+data
+docker-compose

healthdata_influx/.gitignore

@@ -0,0 +1,96 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+data/
+.DS_Store
+config.yml
+!docker-compose/config.yml
+
+.vscode

healthdata_influx/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3
+
+WORKDIR /app
+
+COPY requirements.txt /app
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY healthdata_influx /app
+
+CMD [ "python3", "import.py", "/data/export.xml" ]

healthdata_influx/README.md

@@ -0,0 +1,90 @@
+# healthdata_influx
+Imports Apple Health Data into InfluxDB.
+
+![Grafana Screenshot](https://www.tannr.com/wp-content/uploads/2017/03/grafana.png "Grafana Screenshot")
+Visualizing InfluxDB using [Grafana](https://grafana.com/).
+
+## How to export iOS Health Data
+1. Go to the Health App
+2. Tap the profile image at the top right
+3. Tap "Export Health Data"
+4. Save the `export.zip` file and extract its XML contents (`export.xml`) somewhere accessible by this script.
+
+## Running as a complete service with Docker Compose (bonus Grafana Graphs!)
+
+This is the easiest way to get up and running quickly. This will spin up the importer, an InfluxDB database, and Grafana with a default dashboard ready to go.
+
+#### Requirements:
+
+* [Docker](https://www.docker.com/) with [Docker Compose](https://docs.docker.com/compose/)
+
+#### Installation:
+
+* Create a `data` directory at the project root and add the `export.xml` inside it
+
+#### Building:
+
+`docker-compose build`
+
+#### Running:
+
+1. `docker-compose up` (add `-d` to run in daemon mode)
+2. Access Grafana in your web browser: [http://localhost:3000](http://localhost:3000)
+
+Username: `admin` Password: `admin`
+
+#### Refreshing data:
+
+1. Replace `data/export.xml` with a new version
+2. `docker-compose run importer`
+
+## Running as a Python module or stand-alone script.
+
+#### Requirements:
+	
+* [Python 3](https://www.python.org/)
+* An accessible [InfluxDB](https://www.influxdata.com/) instance
+
+#### Installation:
+
+* `pip install -r requirements.txt`
+* Rename `config_sample.yml` to `config.yml`
+
+#### Configuration:
+
+* Edit `config.yml` to match your InfluxDB settings (host, auth, etc.)
+
+#### Usage:
+
+1. Export Health Data from iOS device
+2. `python3 import.py export.xml`
+
+
+#### See also:
+`python import.py --help`
+
+## Running as a stand-alone Docker container
+
+#### Requirements:
+
+* [Docker](https://www.docker.com/)
+
+#### Installation:
+
+* Create a `data` directory at the repo root and add the `export.xml` inside it.
+ 	* (note that this can be anywhere if the volume mount point on the `docker run` command is changed)
+
+#### Configuration:
+
+* Edit `config.yml` to match your InfluxDB settings (host, auth, etc.)
+
+#### Building:
+
+`docker build . -t twstokes/healthdata_influx`
+
+#### Running (at the repo root):
+
+`docker run -v $PWD/data:/data:ro -v $PWD/config.yml:/app/config.yml:ro twstokes/healthdata_influx`
+
+## Todo / Notes:
+* Does not support "Mindful Sessions"

healthdata_influx/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3'
+
+services:
+  importer:
+    build: .
+    volumes:
+      - ./data:/data:ro
+      - ./docker-compose/config.yml:/app/config.yml:ro

healthdata_influx/docker-compose/config.yml

@@ -0,0 +1,12 @@
+# configure with params from the docs:
+# http://influxdb-python.readthedocs.io/en/latest/api-documentation.html#influxdbclient
+
+influxdb:
+  client:
+    host: home.toozhao.com
+    port: 28086
+    database: home
+    username: junv
+    password: "@wahyd4"
+  write_points:
+    batch_size: 1000

healthdata_influx/docker-compose/grafana_dashboard_providers/default.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'default'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
+  options:
+    path: /var/lib/grafana/dashboards

healthdata_influx/docker-compose/grafana_dashboards/default.json

@@ -0,0 +1,176 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "InfluxDB",
+      "fill": 1,
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "$$hashKey": "object:172",
+          "groupBy": [
+            {
+              "params": [
+                "24h"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "HKQuantityTypeIdentifierStepCount",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Steps",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-2y",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Health Data",
+  "uid": "k0lVpINmz",
+  "version": 1
+}

healthdata_influx/docker-compose/grafana_datasources/grafana_influx.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: InfluxDB
+    type: influxdb
+    access: proxy
+    database: health
+    user: grafana
+    password: grafana
+    url: http://influxdb:8086

healthdata_influx/healthdata_influx/db.py

@@ -0,0 +1,65 @@
+"""Loads a configuration and imports data points into InfluxDB"""
+from datetime import datetime, timezone
+import yaml
+from influxdb import InfluxDBClient
+
+class InfluxDBUploader:
+    """Uploads data points to an InfluxDB instance"""
+    def __init__(self, config_path):
+        self.config = self._load_config(config_path)
+
+    def upload(self, data_points=None):
+        """Uploads data points to InfluxDB"""
+        if data_points is not None and len(data_points):
+            client = InfluxDBClient(**self.config['influxdb']['client'])
+            # only creates a DB if none exists
+            client.create_database(self.config['influxdb']['client']['database'])
+            client.write_points(points=data_points, **self.config['influxdb']['write_points'])
+
+    def create_point(self, measurement, time, fields, tags=None):
+        """Helps enforce proper InfluxDB point creation"""
+        # tags can be an empty dict
+        if tags is None:
+            tags = {}
+
+        if not isinstance(measurement, str):
+            raise TypeError('Measurement must be a string.')
+
+        if not isinstance(time, datetime):
+            raise TypeError('Time must be a datetime object.')
+
+        if not isinstance(fields, dict):
+            raise TypeError('Fields must be a dictionary.')
+        elif len(fields) < 1:
+            # there must be at least one field
+            raise ValueError('Fields must contain at least one field.')
+
+        if not isinstance(tags, dict):
+            raise TypeError('Tags must be a dictionary.')
+
+        # convert datetime object to string
+        time = self._create_influx_time(time)
+
+        point = {
+            'tags': tags,
+            'time': time,
+            'fields': fields,
+            'measurement': measurement
+        }
+
+        return point
+
+    def _load_config(self, config_path):
+        """Loads config for this script"""
+        with open(config_path) as file:
+            config = yaml.safe_load(file)
+        return config
+
+    def _create_influx_time(self, time):
+        """
+        Takes in a datetime object
+        Returns a datetime string InfluxDB expects, in UTC
+        """
+        # save as correct format in UTC timezone
+        converted_time = time.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+        return converted_time

healthdata_influx/healthdata_influx/import.py

@@ -0,0 +1,154 @@
+"""
+Parses an Apple Health export file
+and imports into InfluxDB
+"""
+import sys
+import argparse
+from datetime import datetime
+from lxml import etree
+from db import InfluxDBUploader
+
+class Importer:
+    """
+    Importer parses an Apple Health XML file
+    and uploads Records to a database
+    """
+    def __init__(self, uploader, dry=False, buffer_size=50000):
+        if dry:
+            print('Dry run - no database changes will be made.')
+
+        self.dry = dry
+        self.uploader = uploader
+        # how many records to Buffer before flushing to the database
+        # adjusting this will affect memory consumption
+        self.buffer_size = buffer_size
+
+    def upload(self, points):
+        """
+        Sends points to the uploader if not a dry run.
+        """
+        if not self.dry:
+            self.uploader.upload(points)
+    
+    def parse_and_upload(self, export_path):
+        """
+        Takes InfluxDB configuration and Apple Health Data file paths
+        Uploads to InfluxDB
+        """
+
+        def create_flusher(buffer, size):
+            def flusher(records):
+                print("Flushing {} points to DB. Current total: {}".format(size, records))
+                self.upload(buffer[:size])
+                # clean up
+                del buffer[:size]
+            return flusher
+
+        try:
+            print('Opening export file...')
+            with open(export_path, mode='rb') as file:
+                context = self.get_record_iterator(file)
+
+                point_buffer = []
+                total_records, success_records = (0, 0)
+                flusher = create_flusher(point_buffer, self.buffer_size)
+
+                for idx, (_, record) in enumerate(context):
+                    total_records += 1
+
+                    try:
+                        point = self.mung_record_to_point(record)
+                        point_buffer.append(point)
+                        success_records += 1
+                    except Exception as error:
+                        output_mung_error(error, record, idx+1)
+
+                    if len(point_buffer) > self.buffer_size - 1:
+                        flusher(total_records)
+
+                    # memory cleanup
+                    record.clear()
+                    while record.getprevious() is not None:
+                        del record.getparent()[0]
+
+                # upload the rest
+                self.upload(point_buffer)
+
+            print("Successful uploads: {}".format(success_records))
+            print("Total records: {}".format(total_records))
+        except Exception as error:
+            print('Failure!')
+            print(sys.exc_info())
+            print(error)
+
+    def mung_record_to_point(self, record):
+        """
+        Returns an InfluxDB point for a health record XML element
+        """
+        attr = record.attrib
+
+        if ('endDate' not in attr
+                or 'value' not in attr
+                or 'type' not in attr):
+            raise ValueError('Failed to find all required fields.')
+
+        tags = {}
+        fields = {}
+
+        value = attr['value']
+        end_date = attr['endDate']
+        measurement = attr['type']
+
+        try:
+            # try to convert to a number
+            value = float(value)
+        except ValueError:
+            # carry on as a string
+            pass
+
+        # set the fields
+        fields['value'] = value
+        # convert to datetime obj
+        time = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S %z')
+
+        if 'unit' in attr:
+            tags['unit'] = attr['unit']
+        if 'sourceName' in attr:
+            tags['source'] = attr['sourceName']
+
+        point = self.uploader.create_point(measurement, time, fields, tags)
+
+        return point
+
+    def get_record_iterator(self, file):
+        """
+        Takes in an Apple Health Data
+        export file, returns iterator for Record elements
+        """
+        return etree.iterparse(file, events=('end',), tag='Record')
+
+def output_mung_error(error, record, index):
+    print("Couldn't convert record to point:", error)
+    print(etree.tostring(record))
+    print("Record index: {}".format(index))
+
+if __name__ == '__main__':
+    PARSER = argparse.ArgumentParser(description='Imports Apple Health Data to InfluxDB')
+
+    PARSER.add_argument('--config_path', help='InfluxDB config file path', default='./config.yml')
+    PARSER.add_argument('--dry', help='Dry run - no DB changes', action='store_true', default=False)
+    PARSER.add_argument('export_path', help='Apple Health Data export file path')
+
+    ARGS = PARSER.parse_args()
+
+    try:
+        UPLOADER = InfluxDBUploader(ARGS.config_path)
+        print('InfluxDB uploader loaded.')
+        IMPORTER = Importer(UPLOADER, ARGS.dry)
+        print('Importer loaded.')
+        IMPORTER.parse_and_upload(ARGS.export_path)
+    except FileNotFoundError:
+        print('Could not load InfluxDB configuration file!')
+    except Exception as error:
+        print('Failed to initialize InfluxDB!')
+        print(error)

healthdata_influx/requirements.txt

@@ -0,0 +1,3 @@
+lxml
+pyyaml
+influxdb