docs: misc improved docs, added fetching raw events section (#133)

* fix: update reference of `hostname` to `client_hostname`

* chore: remove unused import

* docs: update versions

* docs: add a link to query page

* docs: create raw_events.py

* docs: add raw data fetching

* docs: minor edit

* docs: comment smartertime importer is deprecated

* docs: additional comments on client.rs

* docs: add wakatime importer

* chore: remove unused import in client.py

* fix: updates to use tuple timeperiods

* docs: replace renaming events with a redaction use case

* docs: better phrasing on aw-import-screentime

Author: 0xbrayo

src/api/rest.rst

@@ -95,5 +95,4 @@ The `heartbeat <heartbeats>` API is one of the most useful endpoints for writing
 
 Query API
 ~~~~~~~~~~~~~
-
-**TODO: Add link to writing queries once that page is done**
+`Writing Queries <./../examples/querying-data.html>`_

src/conf.py

@@ -83,11 +83,11 @@
 # built documents.
 #
 # The short X.Y version.
-version = "0.11"
+version = "0.13"
 
 # The full version, including alpha/beta/rc tags.
 # TODO: Get from aw_server version
-release = "v0.11.0"
+release = "v0.13.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

src/examples/client.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 from time import sleep
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 
 from aw_core.models import Event
 from aw_client import ActivityWatchClient

src/examples/client.rs

@@ -33,6 +33,7 @@ async fn main() {
     let bucket_id = format!("test-client-bucket_{}", aw_client.hostname);
     let event_type = "dummy_data".to_string();
 
+    // Note that in a real application, you would want to handle these errors
     create_bucket(&aw_client, bucket_id.clone(), event_type)
         .await
         .unwrap();
@@ -60,6 +61,7 @@ async fn main() {
             .unwrap();
 
         // Sleep a second until next heartbeat (eventually drifts due to time spent in the loop)
+        // You could use wait on tokio intervals to avoid drift
         tokio::time::sleep(tokio::time::Duration::from_secs_f64(sleeptime)).await;
     }
 

src/examples/query_client.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-from time import sleep
 from datetime import datetime, timedelta, timezone
 
 from aw_core.models import Event
@@ -14,10 +13,15 @@
 start = now
 
 query = "RETURN=0;"
-res = client.query(query, "1970-01-01", "2100-01-01")
+start_date = datetime(1970, 1, 1, tzinfo=timezone.utc)
+end_date = datetime(2100, 1, 1, tzinfo=timezone.utc)
+
+# Note that the timeperiods are a list of tuples
+timeperiods = [(start_date, end_date)] 
+res = client.query(query, timeperiods=timeperiods)
 print(res) # Should print 0
 
-bucket_id = "{}_{}".format("test-client-bucket", client.hostname)
+bucket_id = "{}_{}".format("test-client-bucket", client.client_hostname)
 event_type = "dummydata"
 client.create_bucket(bucket_id, event_type="test")
 
@@ -34,22 +38,23 @@ def insert_events(label: str, count: int):
 
 insert_events("a", 5)
 
-query = "RETURN = query_bucket('{}');".format(bucket_id)
+query = 'RETURN = query_bucket("{}");'.format(bucket_id)
 
-res = client.query(query, "1970", "2100")
+res = client.query(query,timeperiods=timeperiods)
 print(res) # Should print the last 5 events
 
-res = client.query(query, start + timedelta(seconds=1), now - timedelta(seconds=2))
+timeperiods_2 = [(start+timedelta(seconds=1), now-timedelta(seconds=2))]
+res = client.query(query, timeperiods=timeperiods_2)
 print(res) # Should print three events
 
 insert_events("b", 10)
 
 query = """
-events = query_bucket('{}');
-merged_events = merge_events_by_keys(events, 'label');
+events = query_bucket("{}");
+merged_events = merge_events_by_keys(events, ["label"]);
 RETURN=merged_events;
 """.format(bucket_id)
-res = client.query(query, "1970", "2100")
+res = client.query(query, timeperiods=timeperiods)
 # Should print two merged events
 # Event "a" with a duration of 5s and event "b" with a duration of 10s
 print(res)

src/examples/querying-data.rst

@@ -80,7 +80,13 @@ Example including aw-client:
 
 Fetching Raw Events
 -------------------
+It is possible to fetch the raw events from a bucket. This is useful if you want to do your own analysis on the data, or if you want to use the aw-analysis library to do transformations on the data.
 
-**TODO:** Write this section
+Example fetching raw events from the "aw-watcher-window_" bucket:
+    This is an example that you can run in a Python to fetch raw events posted by the window watcher.
+    The scripts sums the time spent on each window title and showcases a data redaction use case.
 
-`Bucket REST API <./rest.html#get-events>`_
+    .. literalinclude:: raw_events.py    
+    
+
+.. TODO `Bucket REST API <./rest.html#get-events>`_

src/examples/raw_events.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+from aw_client import ActivityWatchClient
+from aw_core.models import Event
+import re
+from typing import Pattern, cast, List, Set
+from copy import deepcopy
+
+client = ActivityWatchClient("test-client", testing=True)
+
+# get the buckets
+buckets = client.get_buckets()
+
+bucket_id = "{}_{}".format("aw-watcher-window", client.client_hostname)
+
+# fetches 1000 events from the bucket
+events = client.get_events(bucket_id, limit=1000)
+
+# sum the time spent on each app
+time_spent = dict()
+for event in events:
+    app = event.data["app"]
+    if app in time_spent:
+        time_spent[app] += event.duration
+    else:
+        time_spent[app] = event.duration
+
+print(time_spent)
+
+# sensitive data pattern
+pattern = re.compile(r"Binance|Metamask|TrustWallet|Trust Wallet")
+
+# what to replace sensitive data with
+REDACTED = "REDACTED"
+
+def _redact_event(e: Event, pattern: Pattern) -> Event:
+    e = deepcopy(e)
+    for k, v in e.data.items():
+        if isinstance(v, str):
+            if pattern.findall(v.lower()):
+                e.data[k] = REDACTED
+    return e
+
+def _find_sensitive(el: List[Event], pattern: Pattern) -> Set:
+    sensitive_ids = set()
+    for e in el:
+        if _check_event(e, pattern):
+            sensitive_ids.add(e.id)
+    return sensitive_ids
+
+def _check_event(e: Event, pattern: Pattern) -> bool:
+    for k, v in e.data.items():
+        if isinstance(v, str):
+            if pattern.findall(v.lower()):
+                return True
+    return False
+
+sensitive_ids = _find_sensitive(events, pattern)
+for e in events:
+    print(f"Event id: {e.id}")
+    if e.id in sensitive_ids:
+        e_before = e
+        e = _redact_event(e, pattern)
+        print(f"\nData before: {e_before.data}")
+        print(f"Data after:  {e.data}")
+        
+        client.delete_event(bucket_id, cast(int, e_before.id))
+        client.insert_event(bucket_id, e)
+        print("Redacted event")

src/importers.rst

@@ -4,8 +4,10 @@ Importers
 ActivityWatch can't track everything, so sometimes you might want to import data from another source.
 
 - :gh-aw:`aw-import-ical`, supports importing from ``.ical`` files or syncing with Google Calendar.
-- :gh-aw:`aw-importer-smartertime`, imports from `smartertime`_ (Android time tracker).
-- :gh-aw:`aw-import-screentime`, attempt at importing from macOS's Screen Time (and potentially iOS through syncing)
+- :gh-aw:`aw-importer-smartertime`, imports from `smartertime`_ Useful for importing Android screentime data.(Note ActivityWatch also has an Android app :gh-aw:`aw-android`)
+- :gh-aw:`aw-import-screentime`, attempt at importing from macOS's Screen Time (and potentially iOS through syncing).
+- :gh:`brayo-pip/aw-import-wakatime`, imports from `Wakatime`_ (For tracking time spent programming).
 
 
 .. _smartertime: https://play.google.com/store/apps/details?id=com.smartertime&hl=en
+.. _Wakatime: https://wakatime.com/