hogumathi-app/hogumathi_app/item_collections.py

from dataclasses import asdict, replace

from importlib.util import find_spec

import os
import json


from flask import request, g, jsonify, render_template,  Blueprint, url_for, session

from twitter_v2.api import ApiV2TweetSource
from .view_model import FeedItem, cleandict

from .content_system import get_all_content, register_content_source

twitter_enabled = False
if find_spec('twitter_v2_facade'):
    from twitter_v2_facade.view_model import tweet_model_dc_vm
    twitter_enabled = True
    
youtube_enabled = False
if find_spec('youtube_facade'):
    from youtube_facade import youtube_model, get_youtube_builder
    youtube_enabled = True

DATA_DIR=".data"

item_collections_bp = Blueprint('item_collections', 'item_collections',
    static_folder='static',
    static_url_path='',
    url_prefix='/')


def get_tweet_collection (collection_id):
    with open(f'{DATA_DIR}/collection/{collection_id}.json', 'rt', encoding='utf-8') as f:
        collection = json.loads(f.read())
        
    return collection

def collection_from_card_source (url):
    """
    temp1 = await fetch('http://localhost:5000/notes/cards/search?q=twitter.com/&limit=10').then(r => r.json())

    re = /(http[^\s]+twitter\.com\/[^\/]+\/status\/[\d]+)/ig
    tweetLinks = temp1.cards.map(c => c.card.content).map(c => c.match(re))
    tweetLinks2 = tweetLinks.flat().filter(l => l)
    tweetLinksS = Array.from(new Set(tweetLinks2))

    statusUrls = tweetLinksS.map(s => new URL(s))
    //users = Array.from(new Set(statusUrls.map(s => s.pathname.split('/')[1])))
    ids = Array.from(new Set(statusUrls.map(s => parseInt(s.pathname.split('/')[3]))))
    """
    
    """
    temp1 = JSON.parse(document.body.innerText)
    // get swipe note + created_at + tweet user + tweet ID
    tweetCards = temp1.cards.map(c => c.card).filter(c => c.content.match(re))
    tweets = tweetCards.map(c => ({created_at: c.created_at, content: c.content, tweets: c.content.match(re).map(m => new URL(m))}))
    
    tweets.filter(t => t.tweets.filter(t2 => t2.user.toLowerCase() == 'stephenmpinto').length)
    
    
    // HN
    re = /(http[^\s]+news.ycombinator\.com\/[^\s]+\=[\d]+)/ig
    linkCards = temp1.cards.map(c => c.card).filter(c => c.content.match(re))
    links = linkCards.map(c => ({created_at: c.created_at, content: c.content, links: c.content.match(re).map(m => new URL(m))}))
    
    // YT (I thnk I've already done this one)
    
    """
    
    # more in 2022 twitter report

    return None


def expand_item (item, me, tweets = None, includes = None, yt_videos = None):
    if 'id' in item:
        t = list(filter(lambda t: item['id'] == t.id, tweets))
        
        if not len(t):
            print("no tweet for item: " + item['id'])
            feed_item = FeedItem(
                id = item['id'],
                text = "(Deleted, suspended or blocked)",
                created_at = "",
                handle = "error",
                display_name = "Error"
            )
            # FIXME 1) put this in relative order to the collection
            # FIXME 2) we can use the tweet link to get the user ID...
            
        else:
            t = t[0]
        
            feed_item = tweet_model_dc_vm(includes, t, me)

            note = item.get('note')
            feed_item = replace(feed_item, note = note)
            
    elif 'yt_id' in item:
        yt_id = item['yt_id']
        
        vid = list(filter(lambda v: v['id'] == yt_id, yt_videos))[0]
        feed_item = youtube_model(vid)
    
        note = item.get('note')
        feed_item.update({'note': note})
    
        
    
    return feed_item



# pagination token is the next tweet_ID
@item_collections_bp.get('/collection/<collection_id>.html')
def get_collection_html (collection_id):
    me = request.args.get('me')
    acct = session.get(me)
    
    max_results = int(request.args.get('max_results', 10))
    
    pagination_token = int(request.args.get('pagination_token', 0))
    
    collection = get_tweet_collection(collection_id)
    
    if 'authorized_users' in collection and (not acct or not me in collection['authorized_users']):
        return 'access denied.', 403
    
    items = collection['items'][pagination_token:(pagination_token + max_results)]
    
    if not len(items):
        return 'no tweets', 404
    
    twitter_token = os.environ.get('BEARER_TOKEN')
    if me and me.startswith('twitter:') and acct:
        twitter_token = acct['access_token']
    
    tweet_source = ApiV2TweetSource(twitter_token)
    
    tweet_ids = filter(lambda i: 'id' in i, items)
    tweet_ids = list(map(lambda item: item['id'], tweet_ids))
    tweets_response = tweet_source.get_tweets( tweet_ids, return_dataclass=True )
    
    
    
    yt_ids = filter(lambda i: 'yt_id' in i, items)
    yt_ids = list(map(lambda item: item['yt_id'], yt_ids))
    youtube = get_youtube_builder()
    
    videos_response = youtube.videos().list(id=','.join(yt_ids), part='snippet,contentDetails,liveStreamingDetails,statistics,recordingDetails', maxResults=1).execute()
    
    
    #print(response_json)
    if tweets_response.errors:
        # types:
        # https://api.twitter.com/2/problems/not-authorized-for-resource (blocked or suspended)
        # https://api.twitter.com/2/problems/resource-not-found (deleted)
        #print(response_json.get('errors'))
        for err in tweets_response.errors:
            if not 'type' in err:
                print('unknown error type: ' + str(err))
            elif err['type'] == 'https://api.twitter.com/2/problems/not-authorized-for-resource':
                print('blocked or suspended tweet: ' + err['value'])
            elif err['type'] == 'https://api.twitter.com/2/problems/resource-not-found':
                print('deleted tweet: ' + err['value'])
            else:
                print('unknown error')
            
            print(json.dumps(err, indent=2))
            
    
    includes = tweets_response.includes
    tweets = tweets_response.data
    
    feed_items = list(map(lambda item: expand_item(item, me, tweets, includes, videos_response['items']), items))
    
    if request.args.get('format') == 'json':
        return jsonify({'ids': tweet_ids,
                       'tweets': cleandict(asdict(tweets_response)),
                       'feed_items': feed_items,
                       'items': items,
                       'pagination_token': pagination_token})
    else:
        query = {}
        
        if pagination_token:
            query['next_data_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
        
        if 'HX-Request' in request.headers:
            return render_template('partial/tweets-timeline.html', tweets = feed_items, user = {}, query = query)
        else:
            if pagination_token:
                query['next_page_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
            return render_template('tweet-collection.html', tweets = feed_items, user = {}, query = query)

# pagination token is the next tweet_ID
@item_collections_bp.get('/collections.html')
def get_collections_html ():
    me = request.args.get('me')
    acct = session.get(me)
    
    collections = []
    with os.scandir('.data/collection') as collections_files:
        for collection_file in collections_files:
            if not collection_file.name.endswith('.json'):
                continue
            
            with open(collection_file.path, 'rt', encoding='utf-8') as f:
                coll = json.load(f)
                if 'authorized_users' in coll and (not acct or not me in coll['authorized_users']):
                    continue
                    
                collection_id = collection_file.name[:-len('.json')]
                
                coll_info = dict(
                    collection_id = collection_id,
                    href = url_for('.get_collection_html', collection_id=collection_id)
                )
                
                collections.append(coll_info)
    
    return jsonify(collections)


@item_collections_bp.post('/data/collection/create/from-cards')
def post_data_collection_create_from_cards ():
    """
    // create collection from search, supporting multiple Tweets per card and Tweets in multiple Cards.
    
    
    re = /(https?[a-z0-9\.\/\:]+twitter\.com\/[0-9a-z\_]+\/status\/[\d]+)/ig
    
    temp1 = await fetch('http://localhost:5000/notes/cards/search?q=twitter.com/').then(r => r.json())

    cardMatches = temp1.cards
    .map(cm => Object.assign({}, cm, {tweetLinks: Array.from(new Set(cm.card.content.match(re)))}))
    .filter(cm => cm.tweetLinks && cm.tweetLinks.length)
    .map(cm => Object.assign({}, cm, {tweetUrls: cm.tweetLinks.map(l => new URL(l))}))
    .map(cm => Object.assign({}, cm, {tweetInfos: cm.tweetUrls.map(u => ({user: u.pathname.split('/')[1], tweetId: u.pathname.split('/')[3]}))}));
    
    collectionCards = {}

    cardMatches.forEach(function (cm) {
        if (!cm.tweetLinks.length) { return; }
        cm.tweetInfos.forEach(function (ti) {
            if (!collectionCards[ti.tweetId]) {
                collectionCards[ti.tweetId] = [];
            }
            collectionCards[ti.tweetId].push(cm.card);
        })
    })

    var collectionItems = [];
    Object.entries(collectionCards).forEach(function (e) {
        var tweetId = e[0], cards = e[1];
        var note = cards.map(function (card) {
            return card.created_at + "\n\n" + card.content;
        }).join("\n\n-\n\n");

        collectionItems.push({id: tweetId, note: note, tweet_infos: cm.tweetInfos, card_infos: cards.map(c => 'card#' + c.id)});
    })
    """
    
    collection = {
        'items': [], # described in JS function above
        'authorized_users': [g.twitter_user['id']]
    }
    
    return jsonify(collection)



def expand_item2 (item, me, tweet_contents = None, includes = None, youtube_contents = None):
    if 'id' in item:
        tweets_response = tweet_contents[ 'twitter:tweet:' + item['id'] ]
        tweets = tweets_response.items
        
        t = list(filter(lambda t: item['id'] == t.id, tweets))
        
        if not len(t):
            print("no tweet for item: " + item['id'])
            feed_item = FeedItem(
                id = item['id'],
                text = "(Deleted, suspended or blocked)",
                created_at = "",
                handle = "error",
                display_name = "Error"
            )
            # FIXME 1) put this in relative order to the collection
            # FIXME 2) we can use the tweet link to get the user ID...
            
        else:
            feed_item = t[0]

            note = item.get('note')
            feed_item = replace(feed_item, note = note)
            
    elif 'yt_id' in item:
        yt_id = item['yt_id']
        
        yt_videos = youtube_contents[ 'youtube:video:' + yt_id ]
        
        feed_item = list(filter(lambda v: v['id'] == yt_id, yt_videos))[0]
        
        note = item.get('note')
        feed_item.update({'note': note})
    
        
    
    return feed_item


def get_collection (collection_id, pagination_token=None, max_results=10):
    collection = get_tweet_collection(collection_id)
    
    return collection

register_content_source("collection:", get_collection, id_pattern="([^:]+)")

# pagination token is the next tweet_ID
@item_collections_bp.get('/collection2/<collection_id>.html')
def get_collection2_html (collection_id):
    me = request.args.get('me')
    acct = session.get(me)
    
    max_results = int(request.args.get('max_results', 10))
    
    pagination_token = int(request.args.get('pagination_token', 0))
    
    #collection = get_tweet_collection(collection_id)
    collection = get_content(f'collection:{collection_id}',
        pagination_token=pagination_token,
        max_results=max_results)
    
    if 'authorized_users' in collection and (not acct or not me in collection['authorized_users']):
        return 'access denied.', 403
    
    items = collection['items'][pagination_token:(pagination_token + max_results)]
    
    if not len(items):
        return 'no tweets', 404
    
    tweet_ids = filter(lambda i: 'id' in i, items)
    tweet_ids = list(map(lambda item: 'twitter:tweet:' + item['id'], tweet_ids))
    
    tweet_contents = get_all_content( tweet_ids )
    
    
    yt_ids = filter(lambda i: 'yt_id' in i, items)
    yt_ids = list(map(lambda item: 'youtube:video:' + item['yt_id'], yt_ids))
    
    youtube_contents = get_all_content( yt_ids )
    
    
    includes = None
    
    feed_items = list(map(lambda item: expand_item2(item, me, tweet_contents, includes, youtube_contents), items))
    
    if request.args.get('format') == 'json':
        return jsonify({'ids': tweet_ids,
                       'tweets': cleandict(asdict(tweets_response)),
                       'feed_items': feed_items,
                       'items': items,
                       'pagination_token': pagination_token})
    else:
        query = {}
        
        if pagination_token:
            query['next_data_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
        
        if 'HX-Request' in request.headers:
            return render_template('partial/tweets-timeline.html', tweets = feed_items, user = {}, query = query)
        else:
            if pagination_token:
                query['next_page_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
            return render_template('tweet-collection.html', tweets = feed_items, user = {}, query = query)