release v0.2.1

README.md

@@ -105,11 +105,32 @@ There is no UI to edit Tweet collections, but they can be created with the follo
 If no "authorized_users" is present then the collection is accessible to all users. By default the `base.html` template 
 file contains a link to a collection named `swipe-all2` which should exist in `.data/collection/swipe-all2.json`.
 
+
+### Feeds
+
+This is a preview release, hard coded feeds are available in the `get_posts` method of `feeds_facade.py`.
+It includes a sample of feeds across providers and shows that they can all be represented in the same model
+and merged together into a timeline-like experience.
+
+### YouTube
+
+This is a preview release, hard coded. OAuth authentication works and should be configured on one's own YouTube account.
+'YouTube Data v3' access is required, to read channel info and followers/subscribers. It's one day old, so one really needs
+to go through the code to understand it.
+
+The developer portal provides a client secret file that should be placed in `.data/yt_client_secret.json`.
+
 ### Glitch Deployment
 
 The app detects if it's running in Glitch and will configure itself accordingly.
 Ensure that the Twitter API is configured to allow access to your project name.
 
+## Running
+
+For local deployment, use Python:
+
+* `python hogumathi_app.py` or `python3 hogumathi_app.py`
+
 ## Contributing
 
 Since I intend to retain full copyright and re-license portions of the project I am only able to accept contributions with a copyright assignment, similar to how the Apache project works.

hogumathi_app.py

@@ -0,0 +1,293 @@
+import os
+import sys
+from importlib.util import find_spec
+from configparser import ConfigParser
+
+from flask import Flask, g, redirect, url_for, render_template, jsonify
+from flask_cors import CORS
+
+sys.path.append('.data/extensions')
+
+if find_spec('twitter_v2_facade'):
+    from twitter_v2_facade import twitter_app as twitter_v2
+    import oauth2_login
+    twitter_enabled = True
+else:
+    print('twitter module not found.')
+    twitter_enabled = False
+
+if find_spec('twitter_v2_live_facade'):
+    from twitter_v2_live_facade import twitter_app as twitter_v2_live
+    import oauth2_login
+    twitter_live_enabled = True
+else:
+    print('twitter live module not found.')
+    twitter_live_enabled = False
+
+if find_spec('twitter_archive_facade'):
+    from twitter_archive_facade import twitter_app as twitter_archive
+    archive_enabled = True
+else:
+    print('twitter archive module not found.')
+    archive_enabled = False
+
+if find_spec('mastodon_facade'):
+    from mastodon_facade import twitter_app as mastodon
+    mastodon_enabled = True
+else:
+    print('mastodon module not found.')
+    mastodon_enabled = False
+
+if find_spec('feeds_facade'):
+    from feeds_facade import twitter_app as feeds
+    feeds_enabled = True
+else:
+    print('feeds module not found.')
+    feeds_enabled = False
+
+if find_spec('youtube_facade'):
+    from youtube_facade import youtube_app as youtube
+    youtube_enabled = True
+else:
+    print('youtube module not found.')
+    youtube_enabled = False
+
+if find_spec('email_facade'):
+    from email_facade import messages_facade as messages
+    messages_enabled = True
+else:
+    print('messages module not found.')
+    messages_enabled = False
+
+
+if find_spec('videojs'):
+    from videojs import videojs_bp
+    videojs_enabled = True
+else:
+    print('videojs module not found.')
+    videojs_enabled = False
+
+if find_spec('visjs'):
+    from visjs import visjs_bp
+    visjs_enabled = True
+else:
+    print('messages module not found.')
+    visjs_enabled = False
+
+
+
+#messages_enabled = False
+#twitter_live_enabled = False
+
+add_account_enabled = True
+
+def import_env ():
+    cp = ConfigParser()
+    if os.path.exists('.env'):
+        with open('.env') as stream:
+            cp.read_string('[default]\n' + stream.read())
+            os.environ.update(dict(cp['default']))
+
+
+
+if __name__ == '__main__':
+    glitch_enabled = os.environ.get('PROJECT_DOMAIN') and True
+    
+    if glitch_enabled:
+        t = os.environ.get('BEARER_TOKEN')
+        if t:
+            t = t.replace('%%', '%')
+            os.environ['BEARER_TOKEN'] = t
+            
+    else:
+        import_env()
+
+    PORT = int(os.environ.get('PORT', 5000))
+    HOST = os.environ.get('HOST', '127.0.0.1')
+    
+    archive_enabled = os.environ.get('ARCHIVE_TWEETS_PATH') and True
+    
+    
+    notes_app_url = os.environ.get('NOTES_APP_URL')
+    bedrss_app_url = os.environ.get('NOTES_APP_URL')
+    
+    plots_app_url = os.environ.get('PLOTS_APP_URL')
+    videos_app_url = os.environ.get('VIDEOS_APP_URL')
+    messages_app_url = os.environ.get('MESSAGES_APP_URL')
+    
+    if not os.path.exists('.data'):
+        os.mkdir('.data')
+    
+    if not os.path.exists('.data/cache'):
+        os.mkdir('.data/cache')
+    
+    api = Flask(__name__, static_url_path='')
+    
+    
+    # HACK - environ from .env isn't set yet when the import happens. We should call an init function somewhere.
+    oauth2_login.app_access_token = os.environ.get("BEARER_TOKEN")
+    oauth2_login.app_consumer_key = os.environ.get("TWITTER_CONSUMER_KEY")
+    oauth2_login.app_secret_key = os.environ.get("TWITTER_CONSUMER_SECRET")
+
+
+    @api.before_request
+    def add_config ():
+        g.twitter_enabled = twitter_enabled
+        g.archive_enabled = archive_enabled
+        g.mastodon_enabled = mastodon_enabled
+        g.feeds_enabled = feeds_enabled
+        g.youtube_enabled = youtube_enabled
+        g.messages_enabled = messages_enabled
+        g.twitter_live_enabled = twitter_live_enabled
+        
+        g.add_account_enabled = add_account_enabled
+        
+        
+        g.glitch_enabled = glitch_enabled
+        
+        g.videojs_enabled = videojs_enabled
+        g.visjs_enabled = visjs_enabled
+        
+        if glitch_enabled:
+            g.app_url = 'https://{}.glitch.me'.format( os.environ.get('PROJECT_DOMAIN') )
+        else:
+            g.app_url = 'http://{}:{}'.format('localhost', PORT)
+            
+        if notes_app_url:
+            g.notes_app_url = notes_app_url
+        if bedrss_app_url:
+            g.bedrss_app_url = bedrss_app_url
+        if plots_app_url:
+            g.plots_app_url = plots_app_url
+        if videos_app_url:
+            g.videos_app_url = videos_app_url
+        if messages_app_url:
+            g.messages_app_url = messages_app_url
+    
+    
+    @api.context_processor
+    def inject_config ():
+        config = {}
+        config['twitter_enabled'] = twitter_enabled
+        config['archive_enabled'] = archive_enabled
+        config['mastodon_enabled'] = mastodon_enabled
+        config['feeds_enabled'] = feeds_enabled
+        config['youtube_enabled'] = youtube_enabled
+        config['messages_enabled'] = messages_enabled
+        config['twitter_live_enabled'] = twitter_live_enabled
+        
+        config['add_account_enabled'] = add_account_enabled
+        
+        config['glitch_enabled'] = glitch_enabled
+        
+        config['videojs_enabled'] = videojs_enabled
+        config['visjs_enabled'] = visjs_enabled
+
+        if notes_app_url:
+            config['notes_app_url'] = notes_app_url
+        if bedrss_app_url:
+            config['bedrss_app_url'] = bedrss_app_url
+        
+        if plots_app_url:
+            config['plots_app_url'] = plots_app_url
+        if videos_app_url:
+            config['videos_app_url'] = videos_app_url
+        if messages_app_url:
+            config['messages_app_url'] = messages_app_url
+            
+        
+        return config
+    
+    
+    @api.context_processor
+    def add_nav_items_to_template_context ():
+        nav_items = []
+        
+        route_nav = g.get('route_nav')
+        
+        if route_nav:
+            nav_items += route_nav
+            
+        module_nav = g.get('module_nav')
+        
+        if module_nav:
+            nav_items += module_nav
+            
+        #nav_items.sort(key = lambda ni: ni['order'])
+        
+        return dict(
+            nav_items = nav_items
+        )
+    
+    api.secret_key = os.environ.get('FLASK_SECRET')
+    
+    api.config['TEMPLATES_AUTO_RELOAD'] = True
+    
+    if videojs_enabled:
+        api.register_blueprint(videojs_bp, url_prefix='/lib/videojs')
+        
+    if visjs_enabled:
+        api.register_blueprint(visjs_bp, url_prefix='/lib/visjs')
+
+    api.register_blueprint(twitter_v2, url_prefix='/twitter')
+    
+    if archive_enabled:
+        api.register_blueprint(twitter_archive, url_prefix='/twitter-archive')
+    
+    if mastodon_enabled:
+        api.register_blueprint(mastodon, url_prefix='/mastodon')
+        
+    if feeds_enabled:
+        if not os.path.exists('.data/cache/feeds'):
+            os.mkdir('.data/cache/feeds')
+        
+        api.register_blueprint(feeds, url_prefix='/feeds')
+    
+    if youtube_enabled:
+        api.register_blueprint(youtube, url_prefix='/youtube')
+    
+    if messages_enabled:
+        api.register_blueprint(messages, url_prefix='/messages')
+        
+    if twitter_live_enabled:
+        api.register_blueprint(twitter_v2_live, url_prefix='/twitter-live')
+    
+    CORS(api)
+    
+    @api.get('/login.html')
+    def get_login_html ():
+        opengraph_info = dict(
+            type = 'webpage', # threads might be article
+            url = g.app_url,
+            title = 'Hogumathi',
+            description = 'An app for Twitter, Mastodon, YouTube, etc; Open Source.'
+        )
+        
+        return render_template('login.html', opengraph_info=opengraph_info)
+    
+    @api.get('/')
+    def index ():
+        return redirect(url_for('.get_login_html'))
+    
+    @api.get('/brand/<brand_id>.html')
+    def get_brand_html (brand_id):
+        brands = {
+            'ispoogedaily': {
+                'id': 'ispoogedaily',
+                'display_name': 'iSpooge Daily',
+                'accounts': [
+                    'twitter:14520320',
+                    'mastodon:mastodon.cloud:109271381872332822',
+                    'youtube:UCUMn9G0yzhQWXiRTOmPLXOg'
+                ]
+            }
+        }
+        
+        brand = brands.get(brand_id.lower())
+        
+        if not brand:
+            return '{"error": "no such brand."}', 404
+        
+        return jsonify(brand)
+        
+    api.run(port=PORT, host=HOST)

mastodon_facade.py

@@ -1,6 +1,6 @@
 from configparser import ConfigParser
 import base64
-from flask import json, Response, render_template, request, send_from_directory, Blueprint, url_for, session, redirect
+from flask import json, Response, render_template, request, send_from_directory, Blueprint, url_for, session, redirect, g
 from flask_cors import CORS
 import sqlite3
 import os
@@ -17,6 +17,10 @@ import dateutil.tz
 
 import requests
 
+from mastodon_source import MastodonAPSource
+from mastodon_v2_types import Status
+
+from view_model import FeedItem, FeedItemAction, PublicMetrics
 
 DATA_DIR='.data'
 
@@ -25,54 +29,81 @@ twitter_app = Blueprint('mastodon_facade', 'mastodon_facade',
     static_url_path='',
     url_prefix='/')
 
-
-
-
-def mastodon_model (post_data):
-    # retweeted_by, avi_icon_url, display_name, handle, created_at, text
-    
+@twitter_app.before_request
+def add_module_nav_to_template_context ():
     
-    user = post_data['account']
-    
-    source_url = post_data['url']
-    
-    avi_icon_url = user['avatar']
-    
-    url = url_for('.get_tweet_html', tweet_id=post_data['id'])
+    me = request.args.get('me')
     
-
+    if me and me.startswith('mastodon:'):
+        youtube_user = session[ me ]
+        
+        g.module_nav = [
+            dict(
+                href = url_for('.get_timeline_home_html', me=me),
+                label = 'Public Timeline',
+                order = 100
+            ),
+            dict(
+                href = url_for('.get_bookmarks_html', me=me),
+                label = 'Bookmarks',
+                order = 200
+            )
+        ]
+
+
+
+def mastodon_model_dc_vm (post_data: Status) -> FeedItem:
+    """
+    This is the method we should use. The others should be refactored out.
+    """
+    
+    user = post_data.account
+    
+    source_url = post_data.url
+    avi_icon_url = user.avatar
+    url = url_for('mastodon_facade.get_tweet_html', tweet_id=post_data.id)
+    
+    actions = {
+        'bookmark': FeedItemAction('mastodon_facade.post_tweet_bookmark', {'tweet_id': post_data.id}),
+        'delete_bookmark': FeedItemAction('mastodon_facade.delete_tweet_bookmark', {'tweet_id': post_data.id}),
         
-    t = {
-        'id': post_data['id'],
+        'retweet': FeedItemAction('mastodon_facade.post_tweet_retweet', {'tweet_id': post_data.id})
+    }
+    
+    t = FeedItem(
+        id = post_data.id,
         # hugely not a fan of allowing the server's HTML out. can we sanitize it ourselves?
-        'html': post_data['content'], 
-        'url': url,
-        'source_url': source_url,
-        'created_at': post_data['created_at'],
-        'avi_icon_url': avi_icon_url,
-        'display_name': user['display_name'],
-        'handle': user['acct'],
+        html = post_data.content, 
+        url = url,
+        source_url = source_url,
+        created_at = post_data.created_at,
+        avi_icon_url = avi_icon_url,
+        display_name = user.display_name,
+        handle = user.acct,
         
-        'author_url': url_for('.get_profile_html', user_id = user['id'], me='mastodon:mastodon.cloud:109271381872332822'),
-        'source_author_url':  user['url'],
+        author_url = url_for('mastodon_facade.get_profile_html', user_id = user.id, me='mastodon:mastodon.cloud:109271381872332822'),
+        source_author_url =  user.url,
         
-        'public_metrics': {
-            'reply_count': post_data['replies_count'],
-            'retweet_count': post_data['reblogs_count'],
-            'like_count': post_data['favourites_count']
-        },
+        public_metrics =  PublicMetrics(
+            reply_count = post_data.replies_count,
+            retweet_count = post_data.reblogs_count,
+            like_count = post_data.favourites_count
+        ),
         
-        'activity': post_data
-    }
+        actions = actions,
+        
+        debug_source_data = post_data
+    )
     
 
     return t
-    
-    
 
 def register_app (instance):
+    endpoint_url = g.app_url
+    redirect_uri = endpoint_url + og_url_for('.get_loggedin_html')
+    
     client_name = os.environ.get('MASTODON_CLIENT_NAME')
-    redirect_uri = 'http://localhost:5004/mastodon/logged-in.html'
+
     
     url = f'https://{instance}/api/v1/apps'
     
@@ -200,7 +231,7 @@ def get_logged_in_html ():
     
     session[me] = session_info
     
-    return redirect(url_for('.get_latest_html', me=me))
+    return redirect(url_for('.get_timeline_home_html', me=me))
     
     #return resp.text, resp.status_code
 
@@ -266,8 +297,13 @@ def post_tweet_retweet (tweet_id):
     return resp.text, resp.status_code
 
 @twitter_app.get('/status/<tweet_id>.html')
-def get_tweet_html (tweet_id):
-    return 'tweet: ' + tweet_id
+@twitter_app.get('/statuses.html')
+def get_tweet_html (tweet_id = None):
+    if not tweet_id:
+        ids = request.args.get('ids').split(',')
+        return f'tweets: {ids}'
+    else:
+        return 'tweet: {tweet_id}'
 
 
 @twitter_app.get('/profile/<user_id>.html')
@@ -314,7 +350,7 @@ def get_profile_html (user_id):
     #tweets = tweet_source.get_timeline("public", timeline_params)
     
     max_id = tweets[-1]["id"] if len(tweets) else ''
-    tweets = list(map(mastodon_model, tweets))
+    tweets = list(map(mastodon_model_dc_vm, tweets))
     
     print("max_id = " + max_id)
     
@@ -343,22 +379,15 @@ def get_profile_html (user_id):
 
 @twitter_app.route('/latest.html', methods=['GET'])
 def get_timeline_home_html (variant = "reverse_chronological"):
-    # retweeted_by, avi_icon_url, display_name, handle, created_at, text
     me = request.args.get('me')
     mastodon_user = session[me]
     
+    instance = mastodon_user.get('instance')
     token = mastodon_user.get('access_token')
-    max_id = request.args.get('max_id')
-    
-    # comes from token or session
-    user_id = "ispoogedaily"
-    
-    if not token:
-        print("No token provided or found in environ.")
-        return Response('{"err": "No token."}', mimetype="application/json", status=400) 
     
-    
-    tweet_source = MastodonAPSource("https://mastodon.cloud", token)
+    max_id = request.args.get('max_id')
+
+    tweet_source = MastodonAPSource(f"https://{instance}", token)
     
     timeline_params = {
         'local': True
@@ -367,10 +396,10 @@ def get_timeline_home_html (variant = "reverse_chronological"):
     if max_id:
         timeline_params['max_id'] = max_id
     
-    tweets = tweet_source.get_timeline("public", timeline_params)
+    tweets = tweet_source.get_timeline("public", timeline_params, return_dataclasses=True)
     
-    max_id = tweets[-1]["id"] if len(tweets) else ''
-    tweets = list(map(mastodon_model, tweets))
+    max_id = tweets[-1].id if len(tweets) else ''
+    tweets = list(map(mastodon_model_dc_vm, tweets))
     
     print("max_id = " + max_id)
     
@@ -382,9 +411,7 @@ def get_timeline_home_html (variant = "reverse_chronological"):
             'next_page_url': url_for('.get_timeline_home_html', max_id=max_id, me=me)
         }
     
-    user = {
-            'id': user_id
-        }
+    user = {}
     
     #return Response(response_json, mimetype='application/json')
     
@@ -398,30 +425,19 @@ def get_bookmarks_html ():
     me = request.args.get('me')
     mastodon_user = session[me]
     
-    instance = mastodon_user['instance']
-    access_token = mastodon_user["access_token"]
+    instance = mastodon_user.get('instance')
+    token = mastodon_user.get('access_token')
     
     max_id = request.args.get('max_id')
 
-    headers = {
-       'Authorization': f'Bearer {access_token}'
-    }
-    params = {}
+    tweet_source = MastodonAPSource(f"https://{instance}", token)
     
-    url = f'https://{instance}/api/v1/bookmarks'
-    
-    params = {}
+    tweets = tweet_source.get_bookmarks(max_id=max_id, return_dataclasses=True)
     
-    if max_id:
-        params['max_id'] = max_id
-    
-    resp = requests.get(url, params=params, headers=headers)
-
-    tweets = json.loads(resp.text)
     #tweets = tweet_source.get_timeline("public", timeline_params)
     
-    max_id = tweets[-1]["id"] if len(tweets) else ''
-    tweets = list(map(mastodon_model, tweets))
+    max_id = tweets[-1].id if len(tweets) else ''
+    tweets = list(map(mastodon_model_dc_vm, tweets))
     
     print("max_id = " + max_id)
     
@@ -483,69 +499,3 @@ def delete_tweet_bookmark (tweet_id):
     resp = requests.post(url, headers=headers)
     
     return resp.text, resp.status_code
-
-
-class MastodonAPSource:
-    def __init__ (self, endpoint, token):
-        self.endpoint = endpoint
-        self.token = token
-        super().__init__()
-
-    def get_timeline (self, path = "home", params = {}):
-
-        url = self.endpoint + "/api/v1/timelines/" + path
-        params = {
-            **params
-        }
-        
-        
-        headers = {"Authorization": "Bearer {}".format(self.token)}
-        
-        response = requests.get(url, params=params, headers=headers)
-        
-        print(response)
-        
-        response_json = json.loads(response.text)
-        
-        return response_json
-    
-    # nice parallel is validation
-    # https://github.com/moko256/twitlatte/blob/master/component_client_mastodon/src/main/java/com/github/moko256/latte/client/mastodon/MastodonApiClientImpl.kt
-    
-    def get_mentions_timeline (self):
-        # /api/v1/notifications?exclude_types=follow,favourite,reblog,poll,follow_request
-        return
-        
-    def get_favorited_timeline (self):
-        # /api/v1/notifications?exclude_types=follow,reblog,mention,poll,follow_request
-        return
-        
-    def get_conversations_timeline (self):
-        # /api/v1/conversations
-        return
-        
-    def get_bookmarks_timeline (self):
-        # /account/bookmarks
-        return
-        
-    def get_favorites_timeline (self):
-        # /account/favourites
-        return
-    
-    
-    def get_user_statuses (self, account_id):
-        # /api/v2/search?type=statuses&account_id=
-        # /api/v1/accounts/:id/statuses
-        return
-    
-    def get_statuses (self, ids):
-        # /api/v1/statuses/:id
-        return
-        
-    def get_profile (self, username):
-        # /api/v1/accounts/search
-        return self.search('@' + username, result_type='accounts')
-        
-    def search (q, result_type = None, following = False):
-        # /api/v2/search
-        return

mastodon_source.py

@@ -0,0 +1,159 @@
+from typing import List
+from dataclasses import asdict, replace
+from dacite import from_dict
+
+import json
+
+import requests
+
+from mastodon_v2_types import Status, Conversation, Context
+
+# https://github.com/mastodon/documentation/blob/master/content/en/api/guidelines.md#paginating-through-api-responses-pagination
+class MastodonAPSource:
+    def __init__ (self, endpoint, token):
+        self.endpoint = endpoint
+        self.token = token
+        super().__init__()
+        
+    def get_status_collection (self, path,
+        params = {}, return_dataclasses = False):
+        
+        url = self.endpoint + path
+        params = {
+            **params
+        }
+        
+        headers = {"Authorization": "Bearer {}".format(self.token)}
+        
+        response = requests.get(url, params=params, headers=headers)
+        
+        #print(response)
+        
+        response_json = json.loads(response.text)
+        
+        try:
+            print('trying to force timeline response to Status type')
+            typed_resp = list(map(lambda s: from_dict(data=s, data_class=Status), response_json))
+        except:
+            raise 'error forcing status to type'
+        
+        if return_dataclasses:
+            return typed_resp
+        else:
+            response_json = list(map(lambda s: asdict(s), typed_resp))
+            
+        return response_json
+
+    def get_timeline (self, path = "home", params = {}, return_dataclasses=False):
+        
+        collection_path = "/api/v1/timelines/" + path
+        params = {
+            **params
+        }
+        
+        return self.get_status_collection(collection_path, params = params, return_dataclasses=return_dataclasses)
+        
+    
+    # nice parallel is validation
+    # https://github.com/moko256/twitlatte/blob/master/component_client_mastodon/src/main/java/com/github/moko256/latte/client/mastodon/MastodonApiClientImpl.kt
+    
+    def get_mentions_timeline (self):
+        # /api/v1/notifications?exclude_types=follow,favourite,reblog,poll,follow_request
+        return
+        
+    def get_favorited_timeline (self):
+        # /api/v1/notifications?exclude_types=follow,reblog,mention,poll,follow_request
+        return
+        
+    def get_conversations (self,
+            max_id = None, since_id = None, min_id = None, limit = None
+            ) -> List[Conversation]:
+        """
+        Use get_status_context on the last_status for a conversation.
+        """
+        
+        url = f'{self.instance}/api/v1/conversations'
+        # https://docs.joinmastodon.org/methods/conversations/
+        # Use HTTP Link header for pagination.
+        headers = {
+            'Authorization': f'Bearer {self.token}'
+        }
+        params = cleandict({
+            'max_id': max_id,
+            'since_id': since_id,
+            'min_id': min_id,
+            'limit': limit
+        })
+        
+        resp = requests.get(url, params=params, headers=headers)
+        response_json = json.loads(resp.text)
+        conversations = list(map(lambda c: from_dict(data=c, data_class=Conversation), response_json))
+        
+        return conversations
+    
+    def get_status_context (self, status_id) -> Context:
+        """
+        Returns all parents and direct children.
+        """
+        url = f'{self.instance}/api/v1/statuses/{status_id}/context'
+        headers = {
+            'Authorization': f'Bearer {self.token}'
+        }
+        
+        resp = requests.get(url, headers=headers)
+        response_json = json.loads(resp.text)
+        context = from_dict(data=response_json, data_class=Context)
+        
+        return context
+    
+    def get_bookmarks (self, 
+        max_id=None, return_dataclasses=False):
+        
+        collection_path = '/api/v1/bookmarks'
+        params = {}
+        
+        if max_id:
+            params['max_id'] = max_id
+        
+        return self.get_status_collection(collection_path, params = params, return_dataclasses=return_dataclasses)
+        
+    def get_favorites_timeline (self):
+        # /account/favourites
+        return
+    
+    
+    def get_user_statuses (self, account_id, 
+        max_id = None, return_dataclasses=False):
+        # /api/v2/search?type=statuses&account_id=
+        # /api/v1/accounts/:id/statuses
+        
+        url = f'{self.endpoint}/api/v1/accounts/{account_id}/statuses'
+        headers = {
+           'Authorization': f'Bearer {self.token}'
+        }
+        params = {}
+        
+        if max_id:
+            params['max_id'] = max_id
+        
+        return self.get_status_collection(collection_path, params = params, return_dataclasses=return_dataclasses)
+    
+    def get_statuses (self, ids):
+        # /api/v1/statuses/:id
+        return
+        
+    def get_profile (self, username):
+        # /api/v1/accounts/search
+        return self.search('@' + username, result_type='accounts')
+        
+    def search (q, result_type = None, following = False):
+        # /api/v2/search
+        return
+        
+def cleandict(d):
+    if isinstance(d, dict):
+        return {k: cleandict(v) for k, v in d.items() if v is not None}
+    elif isinstance(d, list):
+        return [cleandict(v) for v in d]
+    else:
+        return d

mastodon_v2_types.py

@@ -0,0 +1,132 @@
+from typing import Optional, List, Dict
+from dataclasses import dataclass
+
+Url = str
+Date = str
+Html = str
+
+StatusId = str
+AccountId = str
+
+@dataclass
+class Account:
+    id: AccountId
+    username: str
+    acct: str
+    url: Url
+    display_name: str
+    
+    note: Html
+    
+    avatar: Url
+    avatar_static: Url
+    
+    header: Url
+    header_static: Url
+    
+    locked: bool
+    
+    fields: List[Dict]
+    emojis: List[Dict]
+    
+    bot: bool
+    group: bool
+    
+    created_at: Date
+    
+    statuses_count: int
+    followers_count: int
+    following_count: int
+    
+    last_status_at: Optional[Date] = None
+    
+    discoverable: Optional[bool] = None
+    noindex: Optional[bool] = None
+    moved: Optional[bool] = None
+    suspended: Optional[bool] = None
+    limited: Optional[bool] = None
+    
+
+Card = Dict
+
+MediaId = str
+BlurHash = str
+
+@dataclass
+class MediaAttachment:
+    """
+    https://docs.joinmastodon.org/entities/MediaAttachment/
+    """
+    id: MediaId
+    type: str
+    url: Url
+    preview_url: Url
+    meta: Dict # Description: Metadata returned by Paperclip.
+    
+    blurhash: BlurHash
+    
+    description: Optional[str] = None # HACK: API doc doesn't say it can be null
+    text_url: Optional[str] = None
+    remote_url: Optional[Url] = None
+    
+
+@dataclass
+class Status:
+    id: StatusId
+    uri: Url
+    created_at: Date
+    account: Account
+    content: Html
+    
+    visibility: str
+    sensitive: bool
+    spoiler_text: str
+    
+    media_attachments: List[MediaAttachment]
+    
+    reblogs_count: int
+    favourites_count: int
+    replies_count: int
+    
+    url: Optional[Url] = None
+    in_reply_to_id: Optional[StatusId] = None
+    in_reply_to_account_id: Optional[AccountId] = None
+    
+    reblog: Optional['Status'] = None
+    
+    card: Optional[Card] = None
+    
+    text: Optional[str] = None
+    
+    edited_at: Optional[Date] = None
+    
+    favourited: Optional[bool] = None
+    reblogged: Optional[bool] = None
+    muted: Optional[bool] = None
+    bookmarked: Optional[bool] = None
+    pinned: Optional[bool] = None
+    filtered: Optional[bool] = None
+    
+
+@dataclass
+class Context:
+    ancestors: List[Status]
+    descendants: List[Status]
+
+ConversationId = str
+
+@dataclass
+class Conversation:
+    id: ConversationId
+    unread: bool
+    accounts: List[Account]
+    last_status: Optional[Status] = None
+
+
+Tag = Dict
+
+@dataclass 
+class Search:
+    accounts: List[Account]
+    statuses: List[Status]
+    hashtags: List[Tag]

oauth2_login.py

@@ -245,7 +245,9 @@ def refresh_token ():
 
 @oauth2_login.route('/refresh-token', methods=['GET'])
 def get_twitter_refresh_token (response_format='json'):
-    return refresh_token()
+    refresh_token()
+    
+    return 'ok'
 
 
 
@@ -272,10 +274,6 @@ def get_twitter_app_refresh_token ():
         
         
     
-    return response.text
+    return 'ok'
+
 
-@oauth2_login.route('/accounts.html', methods=['GET'])
-def get_loggedin_accounts_html ():
-    twitter_accounts = dict(filter(lambda e: e[0].startswith('twitter:'), session.items()))
-    
-    return jsonify(twitter_accounts)

requirements.txt

@@ -1,3 +1,4 @@
+dacite                    ~= 1.6.0
 Flask                     ~= 2.1.2
 Flask-Cors                ~= 3.0.10
 Flask-Session             ~= 0.4.0
@@ -5,4 +6,7 @@ json-stream               ~= 1.3.0
 python-dateutil           ~= 2.8.2
 requests                  ~= 2.28.0
 requests-oauthlib         ~= 1.3.1
-dataclass-type-validator  ~= 0.1.2
+google-api-python-client  ~= 2.70.0
+google-auth               ~= 2.15.0
+google-auth-httplib2      ~= 0.1.0
+google-auth-oauthlib      ~= 0.8.0

sample-env.txt

@@ -17,6 +17,9 @@ TWITTER_CONSUMER_SECRET=
 # Path to tweet.js converted into json (barely works, bolting these two things together)
 ARCHIVE_TWEETS_PATH=
 
+# API key for the app to use if the user has not logged in.
+GOOGLE_DEVELOPER_KEY=
+
 # For development on localhost set value to 1
 #OAUTHLIB_INSECURE_TRANSPORT=1
 #PORT=5004

static/tweets-ui.js

@@ -1,3 +1,46 @@
+function swapVideoPlayer(imgEl, videoUrl, videoType) {
+  
+  if (videoType == 'video/youtube' || videoType == 'video/bitchute') {
+    
+	if (videoType == 'video/youtube') {
+		videoUrl += '?enablejsapi=1&widgetid=1&modestbranding=1&playsinline=true&showinfo=0&autoplay=1&rel=0&mute=0';
+	} else if (videoType == 'video/bitchute') {
+		videoUrl += '?autoplay=1';
+	}
+	
+    var iframe = document.createElement('iframe');
+	iframe.src = videoUrl;
+  iframe.height = imgEl.height;
+  iframe.width = imgEl.width;	
+  iframe.setAttribute('allowfullscreen','allowfullscreen');
+  iframe.setAttribute('allow', 'autoplay');
+    iframe.style.border = '0';
+	imgEl.replaceWith(iframe);
+	
+	
+	
+
+	
+    return iframe;
+  }
+  
+  var vid = document.createElement('video');
+  var src = document.createElement('source');
+  vid.appendChild(src);
+
+  vid.poster = imgEl.src;
+  vid.controls = "controls";
+  vid.height = imgEl.height;
+  vid.width = imgEl.width;
+  vid.playsInline = true;
+  vid.autoplay = true;
+    
+  src.src = videoUrl;
+  src.type = videoType;
+
+  imgEl.replaceWith(vid);
+  return vid;
+}
 
 function noteCardFormat (tweet, annotation) {
 	if (!tweet) { return ""; }

templates/base.html

@@ -3,16 +3,30 @@
 <head>
 	<meta charset="utf-8">
 	<meta name="viewport" content="width=device-width, initial-scale=1">
-	<link rel="stylesheet" href="{{ url_for('.static', filename='tachyons.min.css') }}">
-	<script src="{{ url_for('.static', filename='htmx.js') }}"></script>
-	<script src="{{ url_for('.static', filename='sweetalert2.js') }}"></script>
+	<link rel="stylesheet" href="{{ url_for('static', filename='tachyons.min.css') }}">
+	<script src="{{ url_for('static', filename='htmx.js') }}"></script>
+	<script src="{{ url_for('static', filename='sweetalert2.js') }}"></script>
+	
+	{% if visjs_enabled %}
+	
+	<script src="{{ url_for('visjs.static', filename='vis-timeline-graph2d.min.js') }}"></script>
+	<link href="{{ url_for('visjs.static', filename='vis-timeline-graph2d.min.css') }}" rel="stylesheet" type="text/css" />
+	
+	{% endif %}
 	
 	{% block head %}
 	<title>{{ title | default('No Title') }}</title>
 	{% endblock %}
 	
-	<link rel="stylesheet" href="{{ url_for('.static', filename='theme/base.css') }}">
-	<link rel="stylesheet" href="{{ url_for('.static', filename='theme/user-14520320.css') }}">
+	<link rel="stylesheet" href="{{ url_for('static', filename='theme/base.css') }}">
+	<link rel="stylesheet" href="{{ url_for('static', filename='theme/user-14520320.css') }}">
+	
+	{% if opengraph_info %}
+	{% for prop, content in opengraph_info.items() %}
+	<meta property="og:{{prop}}" content="{{content}}" />
+	{% endfor %}
+	{% endif %}
+	
 <script>
 const Toast = Swal.mixin({
   toast: true,
@@ -26,6 +40,7 @@ const Toast = Swal.mixin({
   }
 })
 </script>
+
 </head>
 <body>
 
@@ -34,28 +49,12 @@ Flexbox makes logical sense but we'll go with a table-based display
 
 -->
 
-<div class="nav" style="position: fixed; width: 25%">
+<div class="nav" style="position: fixed; width: 25%; padding: 4px">
 <ul>
-	{% if twitter_user %}
-	<li><a href="{{ url_for('.get_timeline_home_html') }}">Latest Tweets</a></li>
-	<li><a href="{{ url_for('.get_conversations_html') }}">DMs</a></li>
-	<li><a hx-push-url="/mentions.html?me={{ me }}" hx-get="/twitter/data/mentions/{{ twitter_user.id }}?me={{ me }}" hx-target="#tweets" hx-swap="outerHTML">Mentions</a></li>
-	<!--
-	<li><a hx-get="/twitter/data/thread/1592514352690900992?me={{ me }}" hx-target="#tweets" hx-swap="outerHTML">Test Thread</a></li>
-	<li><a hx-get="/twitter/data/conversation/1592596557009801216?me={{ me }}" hx-target="#tweets" hx-swap="outerHTML">Test Conversation</a></li>
-	<li><a hx-get="/twitter/data/tweets?ids=1592637236147027970,1592474342289330176&amp;me={{ me }}" hx-target="#tweets" hx-swap="outerHTML">Test Tweets</a></li>
-	-->
-	<li><a href="{{ url_for('.get_bookmarks_html') }}">Bookmarks</a></li>
-	<li><a href="{{ url_for('.get_collection_html', collection_id='swipe-all2') }}">Test Collection</a></li>
-	
-	
-	
-	<li><a href="{{ url_for('twitter_v2_facade.oauth2_login.get_logout_html') }}">Logout ({{me}})</a></li
-	{% endif %}
-	
-	{% if mastodon_user %}
-	<li><a href="{{ url_for('.get_timeline_home_html', me=me) }}">Public Timeline</a></li>
-	<li><a href="{{ url_for('.get_bookmarks_html', me=me) }}">Bookmarks</a></li>
+	{% if nav_items %}
+		{% for nav_item in nav_items|sort(attribute='order') %}
+		<li><a href="{{ nav_item.href }}">{{ nav_item.label }}</a></li>
+		{% endfor %}
 	{% endif %}
 	
 	<li><a href="javascript:document.body.classList.toggle('theme-dark')">Toggle Dark Mode</a></li>
@@ -75,6 +74,8 @@ Flexbox makes logical sense but we'll go with a table-based display
 -->
 {% endif %}
 
+<h2>Accounts</h2>
+
 {% include "partial/user-picker.html" %}
 
 {% if add_account_enabled %}

templates/following.html

@@ -1,9 +1,7 @@
 {% extends "base.html" %}
 
 {% block content %}
-<ul>
-{% for follower in following %}
-	<li><a href="{{ url_for('.get_profile_html', user_id=follower) }}">{{ follower }}</a></li>
-{% endfor %}
-</ul>
+
+{% include "partial/users-list.html" %}
+
 {% endblock %}

templates/login.html

@@ -1,3 +1,10 @@
+
+{% extends "base.html" %}
+
+{% block content %}
+
+<div class="w-80" style="background-color: white; border: 1px solid black; padding: 16px; margin: 4px">
+
 <h1>Login</h1>
 
 {% if glitch_enabled %}
@@ -20,9 +27,13 @@
 		<p><a href="mailto:biz@harlanji.com?subject=mastodon%20server">Get your own managed Mastodon server</a></p>
 		</li>
 {% endif %}
-{% if email_enabled %}
-	<li><form method="GET" action="{{ url_for('email_facade.get_login_html') }}">
-		<button type="submit">Email</button> <input name="address" placeholder="me@example.com"></form>
+{% if messages_enabled %}
+	<li><form method="POST" action="{{ url_for('messages_facade.post_api_login') }}">
+		<button type="submit">Email</button> <input name="host" placeholder="smtp.example.com">
+		<input name="user" placeholder="me@example.com">
+		<input name="pass" type="password" placeholder="pa$$word123">
+		</form>
+		<p>This will use IMap and SMTP over SSL.</p>
 		</li>
 {% endif %}
 {% if youtube_enabled %}
@@ -37,4 +48,10 @@
 
 {% endif %}
 
-{% include "partial/user-picker.html" %}
+<h2>Logged In Accounts</h2>
+
+{% include "partial/user-picker.html" %}
+
+</div>
+
+{% endblock %}

templates/partial/compose-form.html

@@ -1,4 +1,10 @@
+
+{% if twitter_user %}
+<form class="compose" hx-post="{{ url_for('twitter_v2_facade.post_tweets_create', me=me) }}" hx-swap="outerHTML">
+{% else %}
 <form class="compose" hx-post="{{ url_for('.post_tweets_create', me=me) }}" hx-swap="outerHTML">
+{% endif %}
+
 <h2>Compose</h2>
 <ul>
 	<li><textarea name="text" placeholder="Only the finest..." style="width: 100%" onchange="this.onkeyup()" onkeyup="this.form.querySelector('.compose-length').innerHTML = this.value.length" rows="6" cols="30"></textarea>

templates/partial/timeline-tweet.html

@@ -32,6 +32,7 @@
 	</div>
 	{% endif %}
 	
+	{% if not skip_embed_replies %}
 	{% if tweet.replied_tweet %}
 	<p style="color: silver">
 	Replying to:
@@ -50,10 +51,13 @@
 	<p style="color: silver">
 	Replying to:
 	</p>
+	{% if tweet.actions.view_replied_tweet %}
 	<p class="reply_to w-100" style="border: 1px solid silver; padding: 6px">
-		<a href="{{ url_for('.get_tweet_html', tweet_id=tweet.replied_tweet_id, view='replies', marked_reply=tweet.id ) }}">View in Thread</a>.
+		<a href="{{ url_for(tweet.actions.view_replied_tweet.route, **tweet.actions.view_replies.route_params) }}">View in Thread</a>.
 	</p>
 	{% endif %}
+	{% endif %}
+	{% endif %}
 	
 	{% if tweet.note %}
 	<p class="note w-100" style="border: 1px solid black; background-color: yellow; padding: 6px">
@@ -77,7 +81,37 @@
 		<p>VIDEOS</p>
 		<ul>
 		{% for video in tweet.videos %}
-			<li><img class="w-100" src="{{ video.preview_image_url }}" referrerpolicy="no-referrer" onclick="this.src='{{ video.image_url }}'"></li>
+			<li><img class="w-100" 
+			       src="{{ video.preview_image_url }}" referrerpolicy="no-referrer"
+				   
+				   
+				   {% if video.image_url %}
+				   onclick="this.src='{{ video.image_url }}'; this.onclick = undefined" 
+				   {% endif %}
+				   
+				   {% if video.url %}
+				   ondblclick="swapVideoPlayer(this, '{{ video.url }}', '{{ video.content_type }}')"
+				   {% endif %}
+				   >
+				   
+				   <dl>
+				    {% if video.duration_str  %}
+					<dt>Duration</dt>
+					<dd>{{ video.duration_str }}</dt>
+					{% elif video.duration_ms  %}
+					<dt>Duration</dt>
+					<dd>{{ video.duration_ms / 1000 / 60 }} minutes</dt>
+					{% endif %}
+				   </dl>
+				   
+				   {% if video.public_metrics and video.public_metrics.view_count %}
+				   <p class="w-100">
+						view count: {{ video.public_metrics.view_count }}
+				   </p>
+				   {% endif %}
+				   
+				   </li>
+				   
 		{% endfor %}
 		</ul>
 
@@ -108,23 +142,69 @@
 
 		<p class="w-100">
 		{% for k, v in tweet.public_metrics.items() %}
+			{% if v != None %}
 			{{ k.replace('_count', 's').replace('ys', 'ies').replace('_', ' ') }}: {{ v }}, 
+			{% endif %}
 		{% endfor %}
 		
 		</p>
 		{% endif %}
 		
+		
+		
 		{% if tweet.non_public_metrics %}
 		
 
 		
 		<p class="w-100">
 			{% for k, v in tweet.non_public_metrics.items() %}
-				{{ k.replace('_count', 's').replace('ys', 'ies').replace('_', ' ') }}: {{ v }}, 
+				{% if v != None %}
+				{{ k.replace('_count', 's').replace('ys', 'ies').replace('_', ' ') }}: {{ v }},
+				{% endif %}				
 			{% endfor %}
 			
 		</p>
 		{% endif %}
+		
+		{% if tweet.attachments %}
+		<ul>
+		{% for a in tweet.attachments %}
+			{% if a.content_type == 'application/vnd-hogumathi.livestream-details+json' %}
+			<li class="livestream-details">
+				<dl>
+					
+					{% if a.content.scheduled_start_time %}
+					<dt>Scheduled Start Time</dt>
+					<dd>{{ a.content.scheduled_start_time }}</dd>
+					{% endif %}
+					
+					{% if a.content.start_time %}
+					<dt>Start Time</dt>
+					<dd>{{ a.content.start_time }}</dd>
+					{% endif %}
+					
+					{% if a.content.chat_embed_url %}
+					<dt>Chat</dt>
+					<dd>
+						<iframe class="w-100" height="400" src="{{ a.content.chat_embed_url }}" referrerpolicy="origin"></iframe>
+					</dd>
+					{% endif %}
+					
+					
+				</dl>
+			</li>
+			{% else %}
+			<li><a href="{{ a.url }}">{{ a.name }}</a> {{ a.content_type }} ({{ a.size }})
+			{% endif %}
+			</li>
+		{% endfor %}
+		</ul>
+		{% endif %}
 	
-	
+		{% if show_conversation_id %}
+		<p class="w-100">
+			Conversation: {{ tweet.conversation_id }}
+			
+		</p>
+		{% endif %}
 </div>

templates/partial/tweets-carousel.html

@@ -1,4 +1,4 @@
-<script src="{{ url_for('.static', filename='tweets-ui.js') }}"></script>
+<script src="{{ url_for('static', filename='tweets-ui.js') }}"></script>
 <script>
 
 {% if notes_app_url %}

templates/partial/tweets-timeline.html

@@ -1,4 +1,8 @@
-<script src="{{ url_for('.static', filename='tweets-ui.js') }}"></script>
+
+
+<script src="{{ url_for('static', filename='tweets-ui.js') }}"></script>
+
+
 <script>
 
 {% if notes_app_url %}
@@ -6,24 +10,112 @@ var notesAppUrl = {{ notes_app_url | tojson }}
 {% endif %}
 
 	if (!window['dataset']) {
+		{% if visjs_enabled %}
+		window.dataset = new vis.DataSet();
+		{% else %}
 		window.dataset = {
 			items: [],
 			update: function (items) {
 				dataset.items = dataset.items.concat(items);
+			},
+			get: function () {
+				return items;
 			}
 		}
+		{% endif %}
 	}
 </script>
 
+{% if twitter_live_enabled and visjs_enabled %}
+
+<div class="w-100" style="position: sticky; top: 20px; background-color: silver; padding: 20px 0; margin: 10px 0;">
+		<div id="visualization"></div>
+	
+</div>
+
+{% endif %}
+
 <ul id="tweets" class="tweets w-75 center z-0">
 
 {% for tweet in tweets %}
 
 <li class="tweet w-100 dt {% if tweet.is_marked %}marked{% endif %}">
 <script>
-	dataset.update([
-		{{ tweet | tojson }}
-	]);
+	function feed_item_to_activity (fi) {
+		
+		var group = 'tweet';
+		var y = 1;
+		if (fi.retweeted_tweet_id) {
+			group = 'retweet';
+			y = 2;
+		} else if (fi.replied_tweet_id) {
+			group = 'reply';
+			y = 3;
+		}
+		
+		return {
+			//'id': fi.id,
+			'x': new Date(fi.created_at),
+			'y': y,
+			'group': group,
+			'feed_item': fi
+		}
+	}
+	
+	function feed_item_to_likes (fi) {
+		
+		if ( !fi['public_metrics'] || !fi.public_metrics['like_count'] ) {
+			return;
+		}
+		
+		var group = 'likes';
+		var y = fi.public_metrics.like_count;
+		
+		
+		return {
+			//'id': fi.id,
+			'x': new Date(fi.created_at),
+			'y': y,
+			'group': group,
+			'feed_item': fi
+		}
+	}
+	
+	function feed_item_to_replies (fi) {
+		
+		if ( !fi['public_metrics'] || !fi.public_metrics['reply_count'] ) {
+			return;
+		}
+		
+		var group = 'replies';
+		var y = fi.public_metrics.reply_count;
+		
+		
+		return {
+			//'id': fi.id,
+			'x': new Date(fi.created_at),
+			'y': y,
+			'group': group,
+			'feed_item': fi
+		}
+	}
+	
+	var feedItem = {{ tweet | tojson }};
+	var plotItems = [];
+	
+	var likesPoint = feed_item_to_likes(feedItem);
+	if (likesPoint) { plotItems.push(likesPoint); }
+	
+	var repliesPoint = feed_item_to_replies(feedItem);
+	if (repliesPoint) { plotItems.push(repliesPoint); }
+	
+	if (!plotItems.length) {
+		plotItems.push(feed_item_to_activity(feedItem))
+	}
+	
+	
+	
+	dataset.update(plotItems);
 
 </script>
 
@@ -46,25 +138,48 @@ var notesAppUrl = {{ notes_app_url | tojson }}
 		
 		<p class="tweet-actions-box">
 		
-		<a href="{{ url_for('.get_tweet_html', tweet_id=tweet.id, view='replies') }}">replies</a>
+		
+		{% if tweet.actions.view_replies %}
+		<a href="{{ url_for(tweet.actions.view_replies.route, **tweet.actions.view_replies.route_params) }}">replies</a>
+		|
+		{% endif %}
+		
+
+		
+		
+		{% if show_thread_controls and tweet.conversation_id %}
+		{% with tweet=tweets[0] %}
+		{% if tweet.actions.view_thread %}
+		<a href="{{ url_for(tweet.actions.view_thread.route, **tweet.actions.view_thread.route_params) }}">author thread</a>
 		|
-		{% if show_thread_controls %}
-		<a href="{{ url_for('.get_tweet_html', tweet_id=tweet.conversation_id, view='thread') }}">author thread</a>
+		{% endif %}
+		{% if tweet.actions.view_conversation %}
+		<a href="{{ url_for(tweet.actions.view_conversation.route, **tweet.actions.view_conversation.route_params) }}">full convo</a>
 		|
-		<a href="{{ url_for('.get_tweet_html', tweet_id=tweet.conversation_id, view='conversation') }}">full convo</a>
+		{% endif %}
+		{% endwith %}
+		{% endif %}
+		
+		{% if tweet.actions.view_activity %}
+		<a href="{{ url_for(tweet.actions.view_activity.route, **tweet.actions.view_activity.route_params) }}">activity</a>
 		|
 		{% endif %}
-		{% if twitter_user %}
-		<a hx-post="{{ url_for('.post_tweet_retweet', tweet_id=tweet.id) }}">retweet</a>
+		
+		{% if tweet.actions.retweet %}
+		<a hx-post="{{ url_for(tweet.actions.retweet.route, **tweet.actions.retweet.route_params) }}">retweet</a>
 		|
 		{% endif %}
-		{% if twitter_user or mastodon_user %}
-		<a hx-post="{{ url_for('.post_tweet_bookmark', tweet_id=tweet.id, me=me) }}">bookmark</a>
+		
+		{% if tweet.actions.bookmark %}
+		<a hx-post="{{ url_for(tweet.actions.bookmark.route, **tweet.actions.bookmark.route_params) }}">bookmark</a>
+		{% if tweet.actions.delete_bookmark %}
 		[
-		<a hx-delete="{{url_for('.delete_tweet_bookmark', tweet_id=tweet.id, me=me)}}">-</a>
+		<a hx-delete="{{ url_for(tweet.actions.delete_bookmark.route, **tweet.actions.delete_bookmark.route_params) }}">-</a>
 		]
+		{% endif %}
 		|
 		{% endif %}
+		
 		<a class="tweet-action copy-formatted" href="javascript:copyTweetToClipboard('{{ tweet.id }}')">copy formatted</a>
 		{% if notes_app_url %}
 		|
@@ -95,15 +210,6 @@ var notesAppUrl = {{ notes_app_url | tojson }}
 		Loading more tweets...
 		</span>
 
-		<script>
-			var profileDataEl = document.querySelector('#profile-data');
-			
-			if (window['dataset'] && profileDataEl) {
-				profileDataEl.innerHTML = dataset.items.filter(i => 'public_metrics' in i).map(i => i.public_metrics.like_count).join(', ');
-			}
-
-
-		</script>
 		</center>
 	</li>
 	
@@ -115,19 +221,121 @@ var notesAppUrl = {{ notes_app_url | tojson }}
 		Go to Next Page
 		</a>
 		
+		
+		</center>
+	
+	</li>
+	
+{% endif %}
+	<li style="display: none">
 		<script>
+			// https://stackoverflow.com/questions/22663353/algorithm-to-remove-extreme-outliers-in-array
+			// we should remove outliers on the X axis. That will mainly be RTs with old dates.
+			// we might also be able to get the date of RT as opposed to OG tweet date.
+			
+			// https://towardsdatascience.com/ways-to-detect-and-remove-the-outliers-404d16608dba
 			var profileDataEl = document.querySelector('#profile-data');
 			
 			if (window['dataset'] && profileDataEl) {
-				profileDataEl.innerHTML = dataset.items.filter(i => 'public_metrics' in i).map(i => i.public_metrics.like_count).join(', ');
+				profileDataEl.innerHTML = dataset.get().filter(i => 'public_metrics' in i).map(i => i.public_metrics.like_count).join(', ');
 			}
 
-
+			{% if visjs_enabled %}
+			if (window.profileActivity) {
+				window.profileActivity.fit()
+			}
+			{% endif %}
 		</script>
-		</center>
-	
 	</li>
+</ul>
+
+			{% if twitter_live_enabled and visjs_enabled %}
+			
+	<script>
+
+
+
+function onClick (e) {
+  // we need to scan the dataset between min/max x/y
+  // 
+  // TODO we want to scale these based on the zoom level / pixel values
+  //
+  // FIXME sometimes we get several points:
+  // We could also go for the closest point within the bound.
+  // Perhaps cycle through upon multiple clicks.
+  // For now we can just zoom in closer.
+  //
+  // range: graph2d.components[3].options.dataAxis.left.range.max
+  // fixing this is lower priority since it is currently static.
+  graph2d = window.profileActivity;
+  
+  var timeWindow = graph2d.getWindow();
+  var windowInSeconds = (timeWindow.end - timeWindow.start) / 1000;
+  
+  var pixelWidth = graph2d.dom.centerContainer.offsetWidth;
+  
+  var secondsPerPixel = windowInSeconds / pixelWidth;
+  
+  console.log('secondsPerPixel = ' + secondsPerPixel);
+  
+  
+  //var MAX_TIME_DIFF  = 1000 * 60 * 60;
+  
+  var MAX_TIME_DIFF = 10 * secondsPerPixel * 1000;
+  var MAX_VALUE_DIFF = 10;
+  
+  console.log(`click. value=${e.value[0]}, time=${e.time}`);
+  console.log(e);
+  
+  var nearbyItems = dataset.get({filter: function (item) {
+	var timeDiff = new Date(item.x).getTime() - e.time.getTime();
+	var valueDiff = item.y - e.value[0];
 	
-{% endif %}
+	return Math.abs(timeDiff) < MAX_TIME_DIFF 
+		   && Math.abs(valueDiff) < MAX_VALUE_DIFF;
+	
+  }});
+  
+  //console.log([e.time, e.value[0]]);
+  console.log('nearby points:');
+  console.log(nearbyItems);
+}
+
+
+var container = document.getElementById('visualization');
+
+
+
+var options = {
+  sort: false,
+  sampling:false,
+  style:'points',
+  dataAxis: {
+		  width: '88px',
 
-</ul>
+	  //visible: false,
+	  left: {
+		  range: {
+			  min: 0, max: 10
+		  }
+	  }
+  },  
+  drawPoints: {
+	  enabled: true,
+	  size: 6,
+	  style: 'circle' // square, circle
+  },
+  defaultGroup: 'Scatterplot',
+  graphHeight: '50px',
+  width: '100%'
+};
+
+var groups = [{id: 'feed_item'}];
+
+window.profileActivity = new vis.Graph2d(container, window.dataset, groups, options);
+
+window.profileActivity.on('click', onClick);
+
+	</script>
+			
+			{% endif %}

templates/partial/user-card.html

@@ -0,0 +1,96 @@
+<div class="user w-100" style="border: 1px solid black; background-color: white; margin: 6px 0; padding: 4px;">
+	
+	<div style="float: right; margin-top: 0; text-align: right">
+		{% if user.profile_image_url %}
+		<img style="float: right; margin-left: 4px;" src="{{ user.profile_image_url }}">
+	{% endif %}
+
+	{% if user.username %}
+	<pre style="display: inline; color: #0f0f0f; font-weight: bold; ">@{{ user.username }}</pre>
+	<br>
+	{% endif %}
+	<span><pre style="display: inline; color: #0f0f0f;">{{ user.id }}</pre></span>
+	
+	</div>
+	
+	<p style="margin-top: 0">
+		
+		
+		{% if user.actions and user.actions.view_profile %}
+		{% with view_profile=user.actions.view_profile %}
+		<a href="{{ url_for( view_profile.route, **view_profile.route_params ) }}">
+		{% endwith %}
+		{% elif user.url %}
+		<a href="{{ user.url }}">
+		{% else %}
+		<a href="{{ url_for('twitter_v2_facade.get_profile_html', user_id=user.id) }}">
+		{% endif %}
+		<strong>{{ user.name }}</strong></a>
+		
+		{% if user.verified %}
+		<small>[verified]</small>
+		{% endif %}
+		
+		{% if user.protected %}
+		<small>[protected]</small>
+		{% endif %}
+	</p>
+	
+		
+	{% if user.description %}
+	<div>
+	
+		{{ user.description }}
+	</div>
+	{% endif %}
+	
+	
+	{% if user.pinned_tweet_id %}
+	<div>
+		<p>Pinned tweet:</p>
+		<div style="border: 1px solid black">{{ user.pinned_tweet_id }}</div>
+	</div>
+	{% endif %}
+	
+	<dl>
+		{% if user.created_at %}
+		<dt>Member since</dt>
+		<dd>{{ user.created_at }}</dd>
+		{% endif %}
+		
+		{% if user.website %}
+		<dt>Website</dt>
+		<dd><a href="{{ user.website }}">{{ user.website }}</a></dd>
+		{% endif %}
+		
+		{% if user.location %}
+		<dt>Location</dt>
+		<dd>{{ user.location }}</dd>
+		{% endif %}
+		
+		{% if user.public_metrics %}
+		
+		{% if user.public_metrics.tweet_count %}
+		<dt>Tweets</dt>
+		<dd>{{ user.public_metrics.tweet_count }}</dd>
+		{% endif %}
+		
+		{% if user.public_metrics.followers_count %}
+		<dt>Followers</dt>
+		<dd>{{ user.public_metrics.followers_count }}</dd>
+		{% endif %}
+		
+		{% if user.public_metrics.following_count %}
+		<dt>Following</dt>
+		<dd>{{ user.public_metrics.following_count }}</dd>
+		{% endif %}
+		
+		{% if user.public_metrics.listed_count %}
+		<dt>Listed</dt>
+		<dd>{{ user.public_metrics.listed_count }}</dd>
+		{% endif %}
+		
+		{% endif %}
+	</dl>
+	
+</div>

templates/partial/user-picker.html

@@ -1,4 +1,4 @@
-<h2>Accounts</h2>
+
 
 <ul>
 	{% for k, v in session.items() %}
@@ -8,9 +8,19 @@
 		{% if mastodon_enabled and k.startswith('mastodon:') %}
 			<li><a href="{{ url_for('mastodon_facade.get_timeline_home_html', me=k) }}">{{ k }}</a>
 		{% endif %}
+		{% if youtube_enabled and k.startswith('youtube:') %}
+			<li><a href="{{ url_for('youtube_facade.get_latest_html', me=k) }}">{{ k }}</a>
+		{% endif %}
+		{% if messages_enabled and k.startswith('messages:') %}
+			<li><a href="{{ url_for('messages_facade.get_latest2_html', me=k) }}">{{ k }}</a>
+		{% endif %}
 	{% endfor %}
 	{% if archive_enabled %}
 		<li><a href="{{ url_for('twitter_archive_facade.get_profile_html', user_id=0) }}">Twitter archive</a>
 	{% endif %}
+	{% if feeds_enabled %}
+	<li><a href="{{ url_for('feeds_facade.get_latest_html', me=None) }}">Feeds</a>
+	{% endif %}
+	
 	
 </ul>

templates/partial/users-list.html

@@ -0,0 +1,9 @@
+
+<ul class="users w-75">
+{% for user in users %}
+	<li style="list-style: none">
+
+	{% include "partial/user-card.html" %}
+	</li>
+{% endfor %}
+</ul>

templates/search.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+
+{% block head %}
+	<title>Tweet Library: {{ user.id }}</title>
+{% endblock %}
+
+
+{% block content %}
+
+<div id="search" class="center z-0 w-50" style="position: sticky; top: 0; background-color: white; padding: 4px 0">
+<center>
+<form method="GET">
+	<input type="text" name="q" value="{{ request.args.q }}" placeholder="Search query...">
+	<button type="submit">Search</button>
+</form>
+</center>
+</div>
+	
+	{% include "partial/tweets-timeline.html" %}
+
+	
+{% endblock %}

templates/tweet-collection.html

@@ -7,18 +7,21 @@
 
 {% block content %}
 
-{% if show_parent_tweet_controls %}
-<div>
-	<a href="{{ url_for('.get_tweet_html', tweet_id=tweets[0].conversation_id, view='thread') }}">author thread</a>
-	|
-	<a href="{{ url_for('.get_tweet_html', tweet_id=tweets[0].conversation_id, view='conversation') }}">full convo</a>
-</div>
-{% endif %}
 
-	{% with show_thread_controls=True %}
+<div class="w-100" style="height: 40px">
 	
-	{% include "partial/tweets-timeline.html" %}
+	<p class="w-100" style="text-align: right">
+		
+		{% if page_nav %}
+		{% for nav_item in page_nav|sort(attribute='order') %}
+		| <a href="{{ nav_item.href }}">{{ nav_item.label }}</a>
+		{% endfor %}
+		{% endif %}
+		
+	</p>
 	
-	{% endwith %}
+</div>
+	
+	{% include "partial/tweets-timeline.html" %}
 	
 {% endblock %}

templates/user-profile.html

@@ -2,36 +2,32 @@
 
 {% block head %}
 	<title>Profile: {{ user.id }}</title>
-{% endblock %}
 
 
+	
+{% endblock %}
+
 {% block content %}
 
-	<div id="profile-data" class="w-100" style="position: fixed; height: 30px; background-color: silver">
-	
-	Data
-	
-	</div>
-	<div id="profile-data" style="position: fixed; top: 30px; background-color: silver; width: 80%">
-		<center>
-		<img src="{{ url_for('.static', filename='fake-tweet-activity.png') }}" alt="fake tweet activity" style="width: 60%; height: 40px;">
-		</center>
-	</div>
-	fake-tweet-activity.png
 
-	<div class="w-100" style="margin-top: 80px">
-		<div class="w-100" style="height: 40px">
+
+	<div class="w-100">
+		{% if twitter_user %}
+		
+		{% include "partial/user-card.html" %}
+		
+		<div class="w-100" style="height: 80px;">
 			<p class="w-100" style="text-align: right">
-			<a href="#">Timeline</a> 
-			|
-			<a href="#">Likes</a> 
-			|
-			<a href="#">Following</a>
-			|
-			<a href="#">Followers</a>
-			|
-			<a href="#">Activity</a>
+			
+			{% if page_nav %}
+			{% for nav_item in page_nav|sort(attribute='order') %}
+			| <a href="{{ nav_item.href }}">{{ nav_item.label }}</a>
+			{% endfor %}
+			{% endif %}
+			
 			</p>
+			
+			<div class="w-100">
 			<form action="{{ url_for('.get_profile_html', user_id=user.id) }}" method="GET">
 			
 			<input type="hidden" name="me" value="{{ me }}">
@@ -43,20 +39,30 @@
 			<br>
 			<button type="submit">Filter</button>
 			</form>
+			</div>
 		</div>
 		
+		
+		
+		{% endif %}
+		
 		{% block tab_content %}
 		
 		{% if tab == "media" %}
 			media
 		
 		{% else %}
+
+
+		
 			{% with show_thread_controls=True %}
 			
 			{% include "partial/tweets-timeline.html" %}
 			
 			{% endwith %}
 			
+
+			
 		{% endif %}
 		
 		{% endblock %}

templates/youtube-channel.html

@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+
+{% block head %}
+	<title>Tweet Library: {{ user.id }}</title>
+{% endblock %}
+
+
+{% block content %}
+
+
+
+<div class="w-100" style="height: 40px">
+	
+	<p class="w-100" style="text-align: right">
+		
+		{% if page_nav %}
+		{% for nav_item in page_nav|sort(attribute='order') %}
+		| <a href="{{ nav_item.href }}">{{ nav_item.label }}</a>
+		{% endfor %}
+		{% endif %}
+		
+	</p>
+	
+</div>
+	
+	{% include "partial/users-list.html" %}
+
+	
+{% endblock %}

tweet_source.py

@@ -1,7 +1,13 @@
+from dataclasses import asdict
+from typing import List
+from dacite import from_dict
+
 import json
 import requests
 import sqlite3
 
+from twitter_v2_types import TweetSearchResponse, DMEventsResponse, UserSearchResponse
+
 class ArchiveTweetSource:
     """
     id, created_at, retweeted, favorited, retweet_count, favorite_count, full_text, in_reply_to_status_id_str, in_reply_to_user_id, in_reply_to_screen_nam
@@ -29,7 +35,7 @@ class ArchiveTweetSource:
         
         # if the ID is not stored as a number (eg. string) then this could be a problem
         if since_id:
-            where_sql.append("id > ?")
+            where_sql.append("cast(id as integer) > ?")
             sql_params.append(since_id)
             
         #if author_id:
@@ -46,7 +52,7 @@ class ArchiveTweetSource:
         if where_sql:
             where_sql = "where {}".format(where_sql)
         
-        sql = "select {} from tweet {} order by created_at asc limit ?".format(sql_cols, where_sql)
+        sql = "select {} from tweet {} order by cast(id as integer) asc limit ?".format(sql_cols, where_sql)
         sql_params.append(max_results)
         
         
@@ -63,21 +69,23 @@ class ArchiveTweetSource:
         return results
     
     def get_tweet (self, id_):
-        return self.get_tweets([id_])
+        tweets = self.get_tweets([id_])
+        if len(tweets):
+            return tweets[0]
     
     def get_tweets (self,
                     ids):
                     
         sql_params = []
         where_sql = []
-        if since_id:
-            ids_in_list_sql = "id in ({})".format( ','.join(['?'] * len(ids)))
-            where_sql.append(ids_in_list_sql)
-            sql_params += ids
         
+        ids_in_list_sql = "id in ({})".format( ','.join(['?'] * len(ids)))
+        where_sql.append(ids_in_list_sql)
+        sql_params += ids
+    
         where_sql = " and ".join(where_sql)
         
-        sql = "select * from tweet where {} limit ?".format(where_sql)
+        sql = "select * from tweet where {}".format(where_sql)
         
         db = self.get_db()
         
@@ -86,6 +94,8 @@ class ArchiveTweetSource:
         
         results = list(map(dict, cur.execute(sql, sql_params).fetchall()))
         
+        results.sort(key=lambda t: ids.index(t['id']))
+        
         return results
     
     def search_tweets (self,
@@ -119,24 +129,49 @@ class TwitterApiV2SocialGraph:
     def __init__ (self, token):
         self.token = token
         
-    def get_user (user_id, is_username=False):
+    def get_user (self, user_id, is_username=False):
         # GET /2/users/:id
         # GET /2/users/by/:username
-        return
+        return self.get_users([user_id], is_username)
     
-    def get_users (user_ids, are_usernames=False):
+    def get_users (self, user_ids, are_usernames=False):
         # GET /2/users/by?usernames=
         # GET /2/users?ids=
-        return
+        
+        
+        
+        user_fields = ["id", "created_at", "name", "username", "location", "profile_image_url", "verified", "description", "public_metrics", "protected", "pinned_tweet_id", "url"]
+        
+        params = {
+            
+            'user.fields' : ','.join(user_fields),
+
+        }
+        
+        if are_usernames:
+            url = "https://api.twitter.com/2/users/by"
+            params['usernames'] = user_ids
+        else:
+            url = "https://api.twitter.com/2/users"
+            params['ids'] = user_ids
+
+        headers = {
+            'Authorization': 'Bearer {}'.format(self.token)
+        }
+        
+        response = requests.get(url, params=params, headers=headers)
+        result = json.loads(response.text)
+        
+        return result
         
     def get_following (self, user_id, 
-                        max_results = 1000, pagination_token = None):
+                        max_results = 50, pagination_token = None, return_dataclass=False):
         # GET /2/users/:id/following
         
         url = "https://api.twitter.com/2/users/{}/following".format(user_id)
         
         
-        user_fields = ["created_at", "name", "username", "location", "profile_image_url", "verified"]
+        user_fields = ["id", "created_at", "name", "username", "location", "profile_image_url", "verified"]
         
         params = {
             'user.fields' : ','.join(user_fields),
@@ -150,25 +185,30 @@ class TwitterApiV2SocialGraph:
             params['pagination_token'] = pagination_token
         
         headers = {
-            'Authorization': 'Bearer {}'.format(self.token),
-            'Content-Type': 'application/json'
+            'Authorization': 'Bearer {}'.format(self.token)
         }
         
         response = requests.get(url, params=params, headers=headers)
         result = json.loads(response.text)
         
+        typed_result = from_dict(data_class=UserSearchResponse, data=result)
+        
+        if return_dataclass:
+            return typed_result
+        
+        result = cleandict(asdict(typed_result))
+        
         return result
         
-        return
         
     def get_followers (self, user_id,
-                        max_results = 1000, pagination_token = None):
+                        max_results = 50, pagination_token = None, return_dataclass=False):
         # GET /2/users/:id/followers
         
         url = "https://api.twitter.com/2/users/{}/followers".format(user_id)
         
         
-        user_fields = ["created_at", "name", "username", "location", "profile_image_url", "verified"]
+        user_fields = ["id", "created_at", "name", "username", "location", "profile_image_url", "verified", "description", "public_metrics", "protected", "pinned_tweet_id", "url"]
         
         params = {
             'user.fields' : ','.join(user_fields),
@@ -182,25 +222,97 @@ class TwitterApiV2SocialGraph:
             params['pagination_token'] = pagination_token
         
         headers = {
-            'Authorization': 'Bearer {}'.format(self.token),
-            'Content-Type': 'application/json'
+            'Authorization': 'Bearer {}'.format(self.token)
         }
         
         response = requests.get(url, params=params, headers=headers)
         result = json.loads(response.text)
         
+        typed_result = from_dict(data_class=UserSearchResponse, data=result)
+        
+        if return_dataclass:
+            return typed_result
+        
+        result = cleandict(asdict(typed_result))
+        
         return result
         
         
-    def follow_user (user_id, target_user_id):
+    def follow_user (self, user_id, target_user_id):
         # POST /2/users/:id/following
         # {target_user_id}
         return
         
-    def unfollow_user (user_id, target_user_id):
+    def unfollow_user (self, user_id, target_user_id):
         # DELETE /2/users/:source_user_id/following/:target_user_id
         return
 
+class ApiV2ConversationSource:
+    def __init__ (self, token):
+        self.token = token
+    
+    def get_recent_events (self, max_results = None, pagination_token = None):
+
+        # https://developer.twitter.com/en/docs/twitter-api/direct-messages/lookup/api-reference/get-dm_events
+        url = "https://api.twitter.com/2/dm_events"
+
+        params = {
+            "dm_event.fields": "id,event_type,text,created_at,dm_conversation_id,sender_id,participant_ids,referenced_tweets,attachments",
+            "expansions": ",".join(["sender_id", "participant_ids"]),
+            
+            "user.fields": ",".join(["id", "created_at", "name", "username", "location", "profile_image_url", "url", "verified"])
+        }
+        
+        if max_results:
+            params['max_results'] = max_results
+        
+        if pagination_token:
+            params['pagination_token'] = pagination_token
+            
+        headers = {"Authorization": "Bearer {}".format(self.token)}
+        
+        response = requests.get(url, params=params, headers=headers)
+        response_json = json.loads(response.text)
+        
+        typed_resp = from_dict(data=response_json, data_class=DMEventsResponse)
+        
+        return typed_resp
+        
+    def get_conversation (self, dm_conversation_id,
+        max_results = None, pagination_token = None):
+        
+        return
+        
+    def get_conversation_with_user (self, user_id,
+        max_results = None, pagination_token = None):
+        
+        return
+    
+    def send_message (self, dm_conversation_id, text, attachments = None):
+        url = f'/2/dm_conversations/{dm_conversation_id}/messages'
+        
+        body = {
+            'text': text
+        }
+        
+        if attachments:
+            body['attachments'] = attachments
+        
+        headers = {"Authorization": "Bearer {}".format(self.token)}
+        
+        resp = requests.post(url, data=json.dumps(body), headers=headers)
+        
+        result = json.loads(resp.text)
+        
+        example_resp_text = """
+        {
+          "dm_conversation_id": "1346889436626259968",
+          "dm_event_id": "128341038123"
+        }
+        """
+        
+        return result 
+        
 class ApiV2TweetSource:
     def __init__ (self, token):
         self.token = token
@@ -289,7 +401,7 @@ class ApiV2TweetSource:
         return result
     
     
-    def get_home_timeline (self, user_id, variant = 'reverse_chronological', max_results = 10, pagination_token = None, since_id = None):
+    def get_home_timeline (self, user_id, variant = 'reverse_chronological', max_results = 10, pagination_token = None, since_id = None, until_id = None, end_time = None) -> TweetSearchResponse:
         """
         Get a user's timeline as viewed by the user themselves.
         """
@@ -297,13 +409,16 @@ class ApiV2TweetSource:
         path = 'users/{}/timelines/{}'.format(user_id, variant)
         
         return self.get_timeline(path, 
-            max_results=max_results, pagination_token=pagination_token, since_id=since_id) 
+            max_results=max_results, pagination_token=pagination_token, since_id=since_id, until_id=until_id, end_time=end_time, return_dataclass=True) 
     
     def get_timeline (self, path,
         max_results = 10, pagination_token = None, since_id = None,
+        until_id = None,
+        end_time = None,
         non_public_metrics = False,
         exclude_replies=False,
-        exclude_retweets=False):
+        exclude_retweets=False,
+        return_dataclass=False):
         """
         Get any timeline, including custom curated timelines built by Tweet Deck / ApiV11.
         """
@@ -345,14 +460,22 @@ class ApiV2TweetSource:
             exclude.append('retweets')
             
         if len(exclude):
+            print(f'get_timeline exclude={exclude}')
             params['exclude'] = ','.join(exclude)
         
         
+        
         if pagination_token:
             params['pagination_token'] = pagination_token
             
         if since_id:
             params['since_id'] = since_id
+            
+        if until_id:
+            params['until_id'] = until_id
+            
+        if end_time:
+            params['end_time'] = end_time
         
         headers = {"Authorization": "Bearer {}".format(token)}
         
@@ -361,21 +484,45 @@ class ApiV2TweetSource:
         response = requests.get(url, params=params, headers=headers)
         response_json = json.loads(response.text)
         
-        return response_json
+        try:
+            #print(json.dumps(response_json, indent = 2))
+            typed_resp = from_dict(data=response_json, data_class=TweetSearchResponse)
+        except:
+            print('error converting response to dataclass')
+            print(json.dumps(response_json, indent = 2))
+            
+            if not return_dataclass:
+                return response_json
+            
+            raise 'error converting response to dataclass'
+        
+        if return_dataclass:
+            return typed_resp
     
+        checked_resp = cleandict(asdict(typed_resp))
+        
+        print('using checked response to get_timeline')
+        
+        #print(json.dumps(checked_resp, indent=2))
+        #print('og=')
+        #print(json.dumps(response_json, indent=2))
+        
+        return checked_resp
+
     def get_mentions_timeline (self, user_id,
-                                max_results = 10, pagination_token = None, since_id = None):
+                                max_results = 10, pagination_token = None, since_id = None, return_dataclass=False):
                                 
         path = "users/{}/mentions".format(user_id)
         
         return self.get_timeline(path, 
-            max_results=max_results, pagination_token=pagination_token, since_id=since_id)
+            max_results=max_results, pagination_token=pagination_token, since_id=since_id, return_dataclass=return_dataclass)
     
     def get_user_timeline (self, user_id,
                           max_results = 10, pagination_token = None, since_id = None,
                           non_public_metrics=False,
                           exclude_replies=False,
-                          exclude_retweets=False):
+                          exclude_retweets=False,
+                          return_dataclass=False):
         """
         Get a user's Tweets as viewed by another.
         """
@@ -384,15 +531,16 @@ class ApiV2TweetSource:
         return self.get_timeline(path, 
             max_results=max_results, pagination_token=pagination_token, since_id=since_id,
             non_public_metrics = non_public_metrics,
-            exclude_replies=exclude_replies, exclude_retweets=exclude_retweets)
+            exclude_replies=exclude_replies, exclude_retweets=exclude_retweets, return_dataclass=return_dataclass)
     
     
-    def get_tweet (self, id_, non_public_metrics = False):
-        return self.get_tweets([id_], non_public_metrics = non_public_metrics)
+    def get_tweet (self, id_, non_public_metrics = False, return_dataclass=False):
+        return self.get_tweets([id_], non_public_metrics = non_public_metrics, return_dataclass=return_dataclass)
     
     def get_tweets (self,
                     ids,
-                    non_public_metrics = False):
+                    non_public_metrics = False,
+                    return_dataclass = False):
                     
         token = self.token
         
@@ -426,7 +574,18 @@ class ApiV2TweetSource:
         response = requests.get(url, params=params, headers=headers)
         response_json = json.loads(response.text)
         
-        return response_json
+        print(json.dumps(response_json, indent=2))
+        
+        typed_resp = from_dict(data=response_json, data_class=TweetSearchResponse)
+        
+        if return_dataclass:
+            return typed_resp
+        
+        checked_resp = cleandict(asdict(typed_resp))
+
+        print('using checked response to search_tweets')
+        
+        return checked_resp
     
     def search_tweets (self,
                        query, 
@@ -434,7 +593,8 @@ class ApiV2TweetSource:
                        since_id = None,
                        max_results = 10,
                        sort_order = None,
-                       non_public_metrics = False
+                       non_public_metrics = False,
+                       return_dataclass = False
                        ):
         
         token = self.token
@@ -480,7 +640,22 @@ class ApiV2TweetSource:
         response = requests.get(url, params=params, headers=headers)
         response_json = json.loads(response.text)
         
-        return response_json
+        try:
+            typed_resp = from_dict(data=response_json, data_class=TweetSearchResponse)
+        except:
+            print('error converting tweet search response to TweetSearchResponse')
+            print(response_json)
+            
+            raise 'error converting tweet search response to TweetSearchResponse'
+            
+        if return_dataclass:
+            return typed_resp
+        
+        checked_resp = cleandict(asdict(typed_resp))
+
+        print('using checked response to search_tweets')
+        
+        return checked_resp
         
         
     
@@ -522,7 +697,8 @@ class ApiV2TweetSource:
                        pagination_token = None,
                        since_id = None,
                        max_results = 10,
-                       sort_order = None
+                       sort_order = None,
+                       return_dataclass=False
                        ):
         
         # FIXME author_id can be determined from a Tweet object
@@ -538,14 +714,16 @@ class ApiV2TweetSource:
         print("get_thread query=" + query)
         
         return self.search_tweets(query, 
-            pagination_token = pagination_token, since_id = since_id, max_results = max_results, sort_order = sort_order)
+            pagination_token = pagination_token, since_id = since_id, max_results = max_results, sort_order = sort_order,
+            return_dataclass=return_dataclass)
     
     def get_bookmarks (self, user_id,
-                          max_results = 10, pagination_token = None, since_id = None):
+                          max_results = 10, pagination_token = None, since_id = None,
+                          return_dataclass=False):
         path = "users/{}/bookmarks".format(user_id)
         
         return self.get_timeline(path, 
-            max_results=max_results, pagination_token=pagination_token, since_id=since_id)
+            max_results=max_results, pagination_token=pagination_token, since_id=since_id, return_dataclass=return_dataclass)
     
     def get_media_tweets (self,
                    author_id = None,
@@ -558,7 +736,8 @@ class ApiV2TweetSource:
                    pagination_token = None,
                    since_id = None,
                    max_results = 10,
-                   sort_order = None
+                   sort_order = None,
+                   return_dataclass=False
                    ):
         
         # FIXME author_id can be determined from a Tweet object
@@ -599,7 +778,7 @@ class ApiV2TweetSource:
             query += "from:{} ".format(author_id)
             
         return self.search_tweets(query, 
-            pagination_token = pagination_token, since_id = since_id, max_results = max_results, sort_order = sort_order)
+            pagination_token = pagination_token, since_id = since_id, max_results = max_results, sort_order = sort_order, return_dataclass = return_dataclass)
     
         
     def get_retweets (self, tweet_id):
@@ -610,19 +789,58 @@ class ApiV2TweetSource:
         # GET /2/tweets/:id/quote_tweets
         return 
         
-    def get_likes (self, user_id,
-                     max_results = 10, pagination_token = None, since_id = None):
+    def get_liked_tweets (self, user_id,
+                     max_results = 10, pagination_token = None, since_id = None, return_dataclass=False):
         # GET /2/users/:id/liked_tweets
+        # User rate limit (User context): 75 requests per 15-minute window per each authenticated user
+        
         path = "users/{}/liked_tweets".format(user_id)
         
         return self.get_timeline(path, 
-            max_results=max_results, pagination_token=pagination_token, since_id=since_id)
+            max_results=max_results, pagination_token=pagination_token, since_id=since_id, return_dataclass=return_dataclass)
         
-    def get_liked_by (self, tweet_id):
+    def get_liking_users (self, tweet_id,
+            max_results = None, pagination_token = None):
         #  GET /2/tweets/:id/liking_users
+        # User rate limit (User context): 75 requests per 15-minute window per each authenticated user
+
+        url = f"https://api.twitter.com/2/tweets/{tweet_id}/liking_users"
+        
+        user_fields = ["id", "created_at", "name", "username", "location", "profile_image_url", "verified", "description", "public_metrics", "protected", "pinned_tweet_id", "url"]
+        
+        expansions = None
+        
+        params = cleandict({
+            "user.fields": ','.join(user_fields),
+            "max_results": max_results,
+            "pagination_token": pagination_token,
+            "expansions": expansions,
+        })
+        
+        headers = {
+            "Authorization": f"Bearer {self.token}"
+        }
+        
+        resp = requests.get(url, headers=headers, params=params)
+        
+        response_json = json.loads(resp.text)
+        
+        return response_json
+        
+    def like_tweet (self, tweet_id):
+        #  POST /2/users/:user_id/likes
+        #  {id: tweet_id}
         return
     
     def get_list_tweets (self, list_id):
         # GET /2/lists/:id/tweets
         return
-    
+    
+    
+def cleandict(d):
+    if isinstance(d, dict):
+        return {k: cleandict(v) for k, v in d.items() if v is not None}
+    elif isinstance(d, list):
+        return [cleandict(v) for v in d]
+    else:
+        return d

twitter_app.py

@@ -1,141 +0,0 @@
-import os
-from importlib.util import find_spec
-from configparser import ConfigParser
-
-from flask import Flask, g, redirect, url_for, render_template, jsonify
-from flask_cors import CORS
-
-
-if find_spec('twitter_v2_facade'):
-    from twitter_v2_facade import twitter_app as twitter_v2
-    import oauth2_login
-    twitter_enabled = True
-else:
-    print('twitter module not found.')
-    twitter_enabled = False
-
-if find_spec('twitter_archive_facade'):
-    from twitter_archive_facade import twitter_app as twitter_archive
-    archive_enabled = True
-else:
-    print('twitter archive module not found.')
-    archive_enabled = False
-
-if find_spec('mastodon_facade'):
-    from mastodon_facade import twitter_app as mastodon
-    mastodon_enabled = True
-else:
-    print('mastodon module not found.')
-    mastodon_enabled = False
-
-add_account_enabled = True
-
-def import_env ():
-    cp = ConfigParser()
-    if os.path.exists('.env'):
-        with open('.env') as stream:
-            cp.read_string('[default]\n' + stream.read())
-            os.environ.update(dict(cp['default']))
-
-
-
-if __name__ == '__main__':
-    import_env()
-
-    PORT = int(os.environ.get('PORT', 5000))
-    HOST = os.environ.get('HOST', '127.0.0.1')
-    
-    archive_enabled = os.environ.get('ARCHIVE_TWEETS_PATH') and True
-    glitch_enabled = os.environ.get('PROJECT_DOMAIN') and True
-    
-    notes_app_url = os.environ.get('NOTES_APP_URL')
-    
-    if not os.path.exists('.data'):
-        os.mkdir('.data')
-    
-    if not os.path.exists('.data/cache'):
-        os.mkdir('.data/cache')
-    
-    api = Flask(__name__, static_url_path='')
-    
-    
-    # HACK - environ from .env isn't set yet when the import happens. We should call an init function somewhere.
-    oauth2_login.app_access_token = os.environ.get("BEARER_TOKEN")
-    oauth2_login.app_consumer_key = os.environ.get("TWITTER_CONSUMER_KEY")
-    oauth2_login.app_secret_key = os.environ.get("TWITTER_CONSUMER_SECRET")
-
-
-    @api.before_request
-    def add_config ():
-        g.twitter_enabled = twitter_enabled
-        g.archive_enabled = archive_enabled
-        g.mastodon_enabled = mastodon_enabled
-        
-        g.add_account_enabled = add_account_enabled
-        
-        
-        g.glitch_enabled = glitch_enabled
-        
-        if glitch_enabled:
-            g.app_url = 'https://{}.glitch.me'.format( os.environ.get('PROJECT_DOMAIN') )
-        else:
-            g.app_url = 'http://{}:{}'.format('localhost', PORT)
-            
-        if notes_app_url:
-            g.notes_app_url = notes_app_url
-        
-
-    
-    @api.context_processor
-    def inject_config ():
-        config = {}
-        config['twitter_enabled'] = twitter_enabled
-        config['archive_enabled'] = archive_enabled
-        config['mastodon_enabled'] = mastodon_enabled
-        
-        config['add_account_enabled'] = add_account_enabled
-        
-        config['glitch_enabled'] = glitch_enabled
-
-        if notes_app_url:
-            config['notes_app_url'] = notes_app_url
-        
-        
-        return config
-    
-    api.secret_key = os.environ.get('FLASK_SECRET')
-    
-    api.config['TEMPLATES_AUTO_RELOAD'] = True
-
-    api.register_blueprint(twitter_v2, url_prefix='/twitter')
-    
-    if archive_enabled:
-        api.register_blueprint(twitter_archive, url_prefix='/twitter-archive')
-    
-    if mastodon_enabled:
-        api.register_blueprint(mastodon, url_prefix='/mastodon')
-    
-    CORS(api)
-    
-    @api.get('/login.html')
-    def get_login_html ():
-        return render_template('login.html')
-    
-    @api.get('/')
-    def index ():
-        return redirect(url_for('.get_login_html'))
-    
-    @api.get('/brand/<brand_id>.html')
-    def get_brand_html (brand_id):
-        brand = {
-            'id': 'ispoogedaily',
-            'display_name': 'iSpooge Daily',
-            'accounts': [
-                'twitter:14520320',
-                'mastodon:mastodon.cloud:109271381872332822'
-            ]
-        }
-        
-        return jsonify(brand)
-        
-    api.run(port=PORT, host=HOST)

twitter_archive_facade.py

@@ -1,3 +1,6 @@
+from typing import List
+from dacite import from_dict
+
 from configparser import ConfigParser
 import base64
 from flask import Flask, json, Response, render_template, request, send_from_directory, Blueprint, url_for, g
@@ -20,6 +23,8 @@ import requests
 
 from tweet_source import ArchiveTweetSource
 
+from view_model import FeedItem, PublicMetrics
+
 ARCHIVE_TWEETS_PATH=os.environ.get('ARCHIVE_TWEETS_PATH', '.data/tweets.json')
 
 
@@ -243,8 +248,8 @@ def tweet_model (tweet_data):
 
     
     t['public_metrics'] = {
-        'like_count': tweet_data['favorite_count'],
-        'retweet_count': tweet_data['retweet_count'],
+        'like_count': int(tweet_data['favorite_count']),
+        'retweet_count': int(tweet_data['retweet_count']),
         'reply_count': 0,
         'quote_count': 0
     }
@@ -252,48 +257,44 @@ def tweet_model (tweet_data):
     return t
 
 
-@twitter_app.route('/data/timeline/user/<user_id>')
-def get_data_timeline_user (user_id):
-
-    pagination_token = request.args.get('pagination_token') # since_id
-    #exclude_replies = request.args.get('exclude_replies')
-    
-    #is_me = user_id == twitter['id']
-    
-    tweet_source = ArchiveTweetSource(ARCHIVE_TWEETS_PATH)
-    
-    db_tweets = tweet_source.get_user_timeline(author_id = user_id,
-                                                    since_id = pagination_token,
-                                                    #exclude_replies = exclude_replies == '1'
-                                                    )
-                                                    
-
-    tweets = list(map(tweet_model, db_tweets))
-    next_token = db_tweets[-1]['id']
+def tweet_model_vm (tweet_data) -> List[FeedItem]:
+    # retweeted_by, avi_icon_url, display_name, handle, created_at, text
     
-    query = {}
+    """
+    {"id": "797839193", "created_at": "2008-04-27T04:00:27", "retweeted": 0, "favorited": 0, "retweet_count": "0", "favorite_count": "0", "full_text": "Putting pizza on. Come over any time!", "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_screen_name": null, "author_id": "14520320"}, {"id": "797849979", "created_at": "2008-04-27T04:27:46", "retweeted": 0, "favorited": 0, "retweet_count": "0", "favorite_count": "0", "full_text": "hijacked!@!!!", "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_screen_name": null, "author_id": "14520320"}
+    """
+    t = FeedItem(
+        id = tweet_data['id'],
+        text = tweet_data['full_text'],
+        created_at = tweet_data['created_at'],
+        author_is_verified = False,
+        
+        conversation_id = tweet_data['id'],
+        
+        avi_icon_url = '',
+        
+        display_name = 'Archive User',
+        handle = '!archive',
+        
+        url = url_for('.get_tweet_html', tweet_id = tweet_data['id']),
+        
+        author_url = url_for('.get_profile_html', user_id='0'),
+        author_id = '0',
+        
+        source_url = '!source_url',
+        source_author_url = '!source_author_url',
+        #'is_edited': len(tweet_data['edit_history_tweet_ids']) > 1
+        
+        public_metrics = PublicMetrics(
+            like_count = int(tweet_data['favorite_count']),
+            retweet_count = int(tweet_data['retweet_count']),
+            reply_count = 0,
+            quote_count = 0
+        )
+    )
     
-    if next_token:
-        query = {
-            **query,
-            
-            'next_data_url': url_for('.get_data_timeline_user', user_id=user_id, pagination_token=next_token)
-        }
+    return t
     
-    if 'HX-Request' in request.headers:
-        user = {
-            'id': user_id
-        }
-        
-        return render_template('partial/tweets-timeline.html', user = user, tweets = tweets, query = query)
-    else:
-        response_body = json.dumps({
-            'tweets': tweets,
-            'query': query
-        })
-        return Response(response_body, mimetype='application/json')
-
-
 
 @twitter_app.route('/profile/<user_id>.html', methods=['GET'])
 def get_profile_html (user_id):
@@ -309,7 +310,7 @@ def get_profile_html (user_id):
                                                     )
                                                     
 
-    tweets = list(map(tweet_model, db_tweets))
+    tweets = list(map(tweet_model_vm, db_tweets))
     next_token = db_tweets[-1]['id']
 
 
@@ -318,26 +319,50 @@ def get_profile_html (user_id):
     if next_token:
         query = {
             **query,
-            
-            'next_data_url': url_for('.get_data_timeline_user', user_id=user_id, pagination_token=next_token, exclude_replies=1),
+            'next_data_url': url_for('.get_profile_html', user_id=user_id , pagination_token=next_token),
             'next_page_url': url_for('.get_profile_html', user_id=user_id , pagination_token=next_token)
         }
     
     profile_user = {
             'id': user_id
         }
-    theme = {
-        'name': None,
-        'body': {'background': 'floralwhite'},
-        'timeline': {'tweet': {'background': 'white',
-                               'border': '1px solid silver'}}
-    }
+
+    if 'HX-Request' in request.headers:
+        user = {
+            'id': user_id
+        }
+        
+        return render_template('partial/tweets-timeline.html', user = profile_user, tweets = tweets, query = query)
+    else:
+        return render_template('user-profile.html', user = profile_user, tweets = tweets, query = query)
+
+@twitter_app.get('/tweet/<tweet_id>.html')
+@twitter_app.get('/tweets.html')
+def get_tweet_html (tweet_id = None):
+    
+    output_format = request.args.get('format')
+    
+    if not tweet_id:
+        ids = request.args.get('ids').split(',')
+    else:
+        ids = [tweet_id]
+        
+    tweet_source = ArchiveTweetSource(ARCHIVE_TWEETS_PATH)
     
-    return render_template('user-profile.html', user = profile_user, tweets = tweets, query = query, theme=theme)
+    db_tweets = tweet_source.get_tweets(ids)
+                                                    
 
-@twitter_app.get('/tweet.html')
-def get_tweet_html (tweet_id):
-    return ''
+    tweets = list(map(tweet_model_vm, db_tweets))
+    query = {}
+    profile_user = {}
+    
+    
+    if output_format == 'feed.json':
+        return jsonify(dict(
+            data = tweets
+            ))
+    else:
+        return render_template('search.html', user = profile_user, tweets = tweets, query = query)
 
 @twitter_app.route('/latest.html', methods=['GET'])
 def get_timeline_home_html (variant = "reverse_chronological", pagination_token=None):
@@ -366,7 +391,7 @@ def post_media_upload ():
 def get_tweets_search (response_format='json'):
     
     search = request.args.get('q')
-    limit = int(request.args.get('limit', 10))
+    limit = int(request.args.get('limit', 10000))
     offset = int(request.args.get('offset', 0))
     
     in_reply_to_user_id = int(request.args.get('in_reply_to_user_id', 0))
@@ -389,6 +414,8 @@ from tweet
         sql += " where in_reply_to_user_id = ?"
         sql_params.append(str(in_reply_to_user_id))
         
+    sql += ' order by cast(id as integer)'
+        
     if limit:
         sql += ' limit ?'
         sql_params.append(limit)
@@ -432,7 +459,11 @@ from tweet
             "metaData": meta,
             "tweets": rows
         }
-        
+    elif response_format == 'html':
+        tweets = list(map(tweet_model_vm, tweets))
+        query = {}
+        profile_user = {}
+        return render_template('search.html', user = profile_user, tweets = tweets, query = query)
     else:
         result = {
             "q": search,
@@ -450,7 +481,7 @@ def post_tweets ():
     
     db = sqlite3.connect('.data/tweet.db')
     
-    db.execute('create table tweet (id, created_at, retweeted, favorited, retweet_count, favorite_count, full_text, in_reply_to_status_id_str, in_reply_to_user_id, in_reply_to_screen_name)')
+    db.execute('create table tweet (id integer, created_at, retweeted, favorited, retweet_count integer, favorite_count integer, full_text, in_reply_to_status_id_str integer, in_reply_to_user_id, in_reply_to_screen_name)')
     db.commit()
     
     i = 0

twitter_v2_types.py

@@ -0,0 +1,190 @@
+from typing import List, Dict, Union, Optional
+from dataclasses import dataclass, asdict
+
+Date = str
+
+Url = str
+ContentType = str
+
+TweetId = str
+UserId = str
+
+MediaKey = str
+
+Error = Dict
+
+@dataclass
+class User:
+    id: UserId
+    
+    username: Optional[str] = None
+    created_at: Optional[Date] = None
+    name: Optional[str] = None
+    verified: Optional[bool] = None
+    profile_image_url: Optional[Url] = None
+    description: Optional[str] = None
+    url: Optional[str] = None
+
+@dataclass
+class PublicMetrics:
+    retweet_count: Optional[int] = None
+    reply_count: Optional[int] = None
+    like_count: Optional[int] = None
+    quote_count: Optional[int] = None
+    
+    # on video media (only?)
+    view_count: Optional[int] = None
+
+@dataclass
+class NonPublicMetrics:
+    impression_count: Optional[int] = None
+    profile_clicks: Optional[int] = None
+    url_link_clicks: Optional[int] = None
+    user_profile_clicks: Optional[int] = None
+    
+@dataclass
+class Media:
+    media_key: MediaKey
+    type: str
+    width: int
+    height: int
+    
+@dataclass
+class PhotoMedia(Media):
+    url: Url
+    
+
+@dataclass
+class VideoMediaVariant:
+    content_type: ContentType
+    url: Url
+    bit_rate: Optional[int] = None
+
+@dataclass
+class VideoMedia (Media):
+    preview_image_url: Url
+    duration_ms: int
+    variants: List[VideoMediaVariant]
+    public_metrics: Optional[PublicMetrics] = None
+    non_public_metrics: Optional[NonPublicMetrics] = None
+
+@dataclass
+class GifMedia (Media):
+    preview_image_url: Url
+    variants: List[VideoMediaVariant]
+    public_metrics: Optional[PublicMetrics] = None
+    non_public_metrics: Optional[NonPublicMetrics] = None
+
+@dataclass
+class TweetAttachments:
+    media_keys: Optional[List[MediaKey]]
+
+@dataclass
+class MentionEntity:
+    start: int
+    end: int
+    id: UserId
+    username: str
+
+@dataclass
+class UrlEntity:
+    start: int
+    end: int
+    url: Url
+    expanded_url: Optional[Url] = None
+    display_url: Optional[str] = None
+    media_key: Optional[MediaKey] = None
+    description: Optional[str] = None
+    title: Optional[str] = None
+    unwound_url: Optional[Url] = None
+    status: Optional[int] = None
+
+@dataclass
+class TweetEntities:
+    urls: Optional[List[UrlEntity]] = None
+    mentions: Optional[List[MentionEntity]] = None
+
+@dataclass 
+class ReferencedTweet:
+    id: TweetId
+    type: str
+
+@dataclass
+class Tweet:
+    id: TweetId
+    created_at: Date
+    text: str
+    
+    conversation_id: Optional[TweetId] = None
+    in_reply_to_tweet_id: Optional[TweetId] = None
+    author_id: Optional[UserId] = None
+    referenced_tweets: Optional[List[ReferencedTweet]] = None
+    entities: Optional[TweetEntities] = None
+    public_metrics: Optional[PublicMetrics] = None
+    non_public_metrics: Optional[NonPublicMetrics] = None
+    attachments: Optional[TweetAttachments] = None
+    edit_history_tweet_ids: Optional[List[TweetId]] = None
+
+
+@dataclass
+class TweetExpansions:
+    users: Optional[List[User]] = None
+    media: Optional[List[Union[VideoMedia, PhotoMedia, GifMedia]]] = None
+    tweets: Optional[List[Tweet]] = None
+
+@dataclass
+class TweetSearchMeta:
+    count: Optional[int] = None # is this an error? different endpoint type?
+    result_count: Optional[int] = None
+    next_token: Optional[str] = None
+    newest_id: Optional[TweetId] = None
+    oldest_id: Optional[TweetId] = None
+
+@dataclass
+class TweetSearchResponse:
+    data: Optional[List[Tweet]] = None
+    meta: Optional[TweetSearchMeta] = None
+    includes: Optional[TweetExpansions] = None
+    errors: Optional[List[Error]] = None
+
+@dataclass
+class DMEvent:
+    id: str
+    created_at: str
+    event_type: str
+    dm_conversation_id: str
+    
+
+@dataclass
+class DMCMessageCreate(DMEvent):
+    sender_id: UserId
+    text: str
+    attachments: Optional[TweetAttachments] = None
+
+@dataclass
+class DMParticpantsJoin (DMEvent):
+    participant_ids: List[UserId]
+    
+
+@dataclass
+class DMParticpantsLeave (DMEvent):
+    participant_ids: List[UserId]
+
+@dataclass
+class DMEventsResponse:
+    data: List[Union[DMCMessageCreate, DMParticpantsJoin, DMParticpantsLeave]]
+    meta: Optional[TweetSearchMeta] = None # is SearchResultsMeta a general response format?
+    includes: Optional[TweetExpansions] = None
+
+
+
+
+UserSearchMeta = Dict
+
+
+@dataclass
+class UserSearchResponse:
+    data: Optional[List[User]] = None
+    meta: Optional[UserSearchMeta] = None
+    includes: Optional[TweetExpansions] = None
+    errors: Optional[List[Error]] = None

view_model.py

@@ -0,0 +1,158 @@
+from dataclasses import dataclass, asdict, replace
+from typing import List, Dict, Optional
+
+@dataclass
+class PublicMetrics:
+    reply_count: Optional[int] = None
+    quote_count: Optional[int] = None
+    retweet_count: Optional[int] = None
+    like_count: Optional[int] = None
+    
+    # may be video only
+    view_count: Optional[int] = None
+    
+    
+    def items (self):
+        return asdict(self).items()
+    
+@dataclass
+class NonPublicMetrics:
+    impression_count: Optional[int] = None
+    user_profile_clicks: Optional[int] = None
+    url_link_clicks: Optional[int] = None
+    
+    def items (self):
+        return asdict(self).items()
+
+@dataclass
+class MediaItem:
+    type: str
+    preview_image_url: str
+    
+    media_key: Optional[str] = None
+    image_url: Optional[str] = None
+    
+    url: Optional[str] = None
+    content_type: Optional[str] = None
+    duration_ms: Optional[int] = None
+    
+    height: Optional[int] = None
+    width: Optional[int] = None
+    
+    size: Optional[int] = None
+    
+    public_metrics: Optional[PublicMetrics] = None
+
+@dataclass
+class Card:
+    display_url: Optional[str] = None
+    source_url: Optional[str] = None
+    content: Optional[str] = None
+    title: Optional[str] = None
+
+
+@dataclass
+class FeedItemAction:
+    route: str
+    route_params: Dict
+
+@dataclass
+class FeedItemAttachment:
+    name: str
+    content_type: str
+    
+    url: Optional[str] = None
+    content: Optional[object] = None
+    
+    size: Optional[int] = None
+
+@dataclass
+class FeedItem:
+    id: str
+    created_at: str
+    
+    display_name: str
+    handle: str
+    
+    text: Optional[str] = None
+    html: Optional[str] = None
+    
+    author_is_verified: Optional[bool] = None
+    url: Optional[str] = None
+    conversation_id: Optional[str] = None
+    
+    avi_icon_url: Optional[str] = None
+    
+    author_url: Optional[str] = None
+    author_id: Optional[str] = None
+    
+    source_url: Optional[str] = None
+    source_author_url: Optional[str] = None
+    
+    reply_depth: Optional[int] = 0
+    
+    is_marked: Optional[bool] = None
+    
+    card: Optional[Card] = None
+    
+    public_metrics: Optional[PublicMetrics] = None
+    non_public_metrics: Optional[NonPublicMetrics] = None
+    
+    retweeted_tweet_id: Optional[str] = None
+    
+    source_retweeted_by_url: Optional[str] = None
+    retweeted_by: Optional[str] = None
+    retweeted_by_url: Optional[str] = None
+    
+    videos: Optional[List[MediaItem]] = None
+    photos: Optional[List[MediaItem]] = None
+    
+    quoted_tweet_id: Optional[str] = None
+    quoted_tweet: Optional['FeedItem'] = None
+    
+    replied_tweet_id: Optional[str] = None
+    replied_tweet: Optional['FeedItem'] = None
+    
+    note: Optional[str] = None
+    
+    debug_source_data: Optional[Dict] = None
+    
+    attachments: Optional[List[FeedItemAttachment]] = None
+    
+    # At some point we may move to feed_item_actions when the set is known
+    actions: Optional[Dict[str, FeedItemAction]] = None
+
+# tm = FeedItem(id="1", text="aa", created_at="fs", display_name="fda", handle="fdsafas")
+
+
+@dataclass
+class FeedServiceUser:
+    id: str
+    url: str
+    name: str # display_name
+    username: str # handle
+    
+    created_at: Optional[str] = None
+    description: Optional[str] = None
+    preview_image_url: Optional[str] = None
+    website: Optional[str] = None
+    
+    actions: Optional[Dict[str, FeedItemAction]] = None 
+    
+    source_url: Optional[str] = None
+    
+    @property
+    def display_name (self) -> str:
+        return self.name
+        
+    @property
+    def handle (self) -> str:
+        return self.username
+
+def cleandict(d):
+    if isinstance(d, dict):
+        return {k: cleandict(v) for k, v in d.items() if v is not None}
+    elif isinstance(d, list):
+        return [cleandict(v) for v in d]
+    else:
+        return d