|
|
@@ -2,6 +2,8 @@ from typing import List
|
|
|
from dataclasses import asdict, replace
|
|
|
from dacite import from_dict
|
|
|
|
|
|
+from importlib.util import find_spec
|
|
|
+
|
|
|
from configparser import ConfigParser
|
|
|
import base64
|
|
|
import sqlite3
|
|
|
@@ -32,10 +34,13 @@ from flask_cors import CORS
|
|
|
from tweet_source import ApiV2TweetSource, TwitterApiV2SocialGraph, ApiV2ConversationSource
|
|
|
from twitter_v2_types import Tweet, TweetExpansions
|
|
|
|
|
|
-from view_model import FeedItem, FeedServiceUser, FeedItemAction, MediaItem, Card, PublicMetrics, NonPublicMetrics, cleandict
|
|
|
+from view_model import FeedItem, FeedServiceUser, ThreadItem, FeedItemAction, MediaItem, Card, PublicMetrics, NonPublicMetrics, UnrepliedSection, cleandict
|
|
|
|
|
|
import oauth2_login
|
|
|
|
|
|
+if find_spec('brands'):
|
|
|
+ from brands import find_brand_by_account, fetch_brand_info
|
|
|
+
|
|
|
DATA_DIR='.data'
|
|
|
|
|
|
twitter_app = Blueprint('twitter_v2_facade', 'twitter_v2_facade',
|
|
|
@@ -167,196 +172,6 @@ class TwitterMetadata:
|
|
|
return tweet
|
|
|
|
|
|
|
|
|
-def collection_from_card_source (url):
|
|
|
- """
|
|
|
- temp1 = await fetch('http://localhost:5000/notes/cards/search?q=twitter.com/&limit=10').then(r => r.json())
|
|
|
-
|
|
|
- re = /(http[^\s]+twitter\.com\/[^\/]+\/status\/[\d]+)/ig
|
|
|
- tweetLinks = temp1.cards.map(c => c.card.content).map(c => c.match(re))
|
|
|
- tweetLinks2 = tweetLinks.flat().filter(l => l)
|
|
|
- tweetLinksS = Array.from(new Set(tweetLinks2))
|
|
|
-
|
|
|
- statusUrls = tweetLinksS.map(s => new URL(s))
|
|
|
- //users = Array.from(new Set(statusUrls.map(s => s.pathname.split('/')[1])))
|
|
|
- ids = Array.from(new Set(statusUrls.map(s => parseInt(s.pathname.split('/')[3]))))
|
|
|
- """
|
|
|
-
|
|
|
- """
|
|
|
- temp1 = JSON.parse(document.body.innerText)
|
|
|
- // get swipe note + created_at + tweet user + tweet ID
|
|
|
- tweetCards = temp1.cards.map(c => c.card).filter(c => c.content.match(re))
|
|
|
- tweets = tweetCards.map(c => ({created_at: c.created_at, content: c.content, tweets: c.content.match(re).map(m => new URL(m))}))
|
|
|
-
|
|
|
- tweets.filter(t => t.tweets.filter(t2 => t2.user.toLowerCase() == 'stephenmpinto').length)
|
|
|
-
|
|
|
-
|
|
|
- // HN
|
|
|
- re = /(http[^\s]+news.ycombinator\.com\/[^\s]+\=[\d]+)/ig
|
|
|
- linkCards = temp1.cards.map(c => c.card).filter(c => c.content.match(re))
|
|
|
- links = linkCards.map(c => ({created_at: c.created_at, content: c.content, links: c.content.match(re).map(m => new URL(m))}))
|
|
|
-
|
|
|
- // YT (I thnk I've already done this one)
|
|
|
-
|
|
|
- """
|
|
|
-
|
|
|
- # more in 2022 twitter report
|
|
|
-
|
|
|
- return None
|
|
|
-
|
|
|
-def get_tweet_collection (collection_id):
|
|
|
- with open(f'{DATA_DIR}/collection/{collection_id}.json', 'rt', encoding='utf-8') as f:
|
|
|
- collection = json.loads(f.read())
|
|
|
-
|
|
|
- return collection
|
|
|
-
|
|
|
-# pagination token is the next tweet_ID
|
|
|
-@twitter_app.get('/collection/<collection_id>.html')
|
|
|
-def get_collection_html (collection_id):
|
|
|
-
|
|
|
- max_results = int(request.args.get('max_results', 10))
|
|
|
-
|
|
|
- pagination_token = request.args.get('pagination_token')
|
|
|
-
|
|
|
- collection = get_tweet_collection(collection_id)
|
|
|
-
|
|
|
- if 'authorized_users' in collection and g.twitter_user['id'] not in collection['authorized_users']:
|
|
|
- return 'access denied.', 403
|
|
|
-
|
|
|
- items = []
|
|
|
- for item in collection['items']:
|
|
|
- tweet_id = item['id']
|
|
|
- if pagination_token and tweet_id != pagination_token:
|
|
|
- continue
|
|
|
- elif tweet_id == pagination_token:
|
|
|
- pagination_token = None
|
|
|
- elif len(items) == max_results:
|
|
|
- pagination_token = tweet_id
|
|
|
- break
|
|
|
-
|
|
|
- items.append(item)
|
|
|
-
|
|
|
-
|
|
|
- if not len(items):
|
|
|
- return 'no tweets', 404
|
|
|
-
|
|
|
- token = g.twitter_user['access_token']
|
|
|
-
|
|
|
-
|
|
|
- tweet_source = ApiV2TweetSource(token)
|
|
|
-
|
|
|
- tweet_ids = list(map(lambda item: item['id'], items))
|
|
|
- response_json = tweet_source.get_tweets( tweet_ids, return_dataclass=True )
|
|
|
-
|
|
|
- #print(response_json)
|
|
|
- if response_json.errors:
|
|
|
- # types:
|
|
|
- # https://api.twitter.com/2/problems/not-authorized-for-resource (blocked or suspended)
|
|
|
- # https://api.twitter.com/2/problems/resource-not-found (deleted)
|
|
|
- #print(response_json.get('errors'))
|
|
|
- for err in response_json.errors:
|
|
|
- if not 'type' in err:
|
|
|
- print('unknown error type: ' + str(err))
|
|
|
- elif err['type'] == 'https://api.twitter.com/2/problems/not-authorized-for-resource':
|
|
|
- print('blocked or suspended tweet: ' + err['value'])
|
|
|
- elif err['type'] == 'https://api.twitter.com/2/problems/resource-not-found':
|
|
|
- print('deleted tweet: ' + err['value'])
|
|
|
- else:
|
|
|
- print('unknown error')
|
|
|
-
|
|
|
- print(json.dumps(err, indent=2))
|
|
|
-
|
|
|
-
|
|
|
- includes = response_json.includes
|
|
|
- tweets = list(map(lambda t: tweet_model_dc_vm(includes, t, g.me), response_json.data))
|
|
|
-
|
|
|
- for item in items:
|
|
|
- t = list(filter(lambda t: item['id'] == t.id, tweets))
|
|
|
-
|
|
|
- if not len(t):
|
|
|
- print("no tweet for item: " + item['id'])
|
|
|
- t = FeedItem(
|
|
|
- id = item['id'],
|
|
|
- text = "(Deleted, suspended or blocked)",
|
|
|
- created_at = "",
|
|
|
- handle = "error",
|
|
|
- display_name = "Error"
|
|
|
- )
|
|
|
- # FIXME 1) put this in relative order to the collection
|
|
|
- # FIXME 2) we can use the tweet link to get the user ID...
|
|
|
-
|
|
|
- else:
|
|
|
- t = t[0]
|
|
|
-
|
|
|
- t = replace(t, note = item['note'])
|
|
|
-
|
|
|
- tweets.append(t)
|
|
|
-
|
|
|
- if request.args.get('format') == 'json':
|
|
|
- return jsonify({'ids': tweet_ids,
|
|
|
- 'data': cleandict(asdict(response_json)),
|
|
|
- 'tweets': tweets,
|
|
|
- 'items': items,
|
|
|
- 'pagination_token': pagination_token})
|
|
|
- else:
|
|
|
- query = {}
|
|
|
-
|
|
|
- if pagination_token:
|
|
|
- query['next_data_url'] = url_for('twitter_v2_facade.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
|
|
|
-
|
|
|
- if 'HX-Request' in request.headers:
|
|
|
- return render_template('partial/tweets-timeline.html', tweets = tweets, user = {}, query = query)
|
|
|
- else:
|
|
|
- if pagination_token:
|
|
|
- query['next_page_url'] = url_for('twitter_v2_facade.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
|
|
|
- return render_template('user-profile.html', tweets = tweets, user = {}, query = query)
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-@twitter_app.post('/data/collection/create/from-cards')
|
|
|
-def post_data_collection_create_from_cards ():
|
|
|
- """
|
|
|
- // create collection from search, supporting multiple Tweets per card and Tweets in multiple Cards.
|
|
|
-
|
|
|
-
|
|
|
- re = /(https?[a-z0-9\.\/\:]+twitter\.com\/[0-9a-z\_]+\/status\/[\d]+)/ig
|
|
|
-
|
|
|
- temp1 = await fetch('http://localhost:5000/notes/cards/search?q=twitter.com/').then(r => r.json())
|
|
|
-
|
|
|
- cardMatches = temp1.cards
|
|
|
- .map(cm => Object.assign({}, cm, {tweetLinks: Array.from(new Set(cm.card.content.match(re)))}))
|
|
|
- .filter(cm => cm.tweetLinks && cm.tweetLinks.length)
|
|
|
- .map(cm => Object.assign({}, cm, {tweetUrls: cm.tweetLinks.map(l => new URL(l))}))
|
|
|
- .map(cm => Object.assign({}, cm, {tweetInfos: cm.tweetUrls.map(u => ({user: u.pathname.split('/')[1], tweetId: u.pathname.split('/')[3]}))}));
|
|
|
-
|
|
|
- collectionCards = {}
|
|
|
-
|
|
|
- cardMatches.forEach(function (cm) {
|
|
|
- if (!cm.tweetLinks.length) { return; }
|
|
|
- cm.tweetInfos.forEach(function (ti) {
|
|
|
- if (!collectionCards[ti.tweetId]) {
|
|
|
- collectionCards[ti.tweetId] = [];
|
|
|
- }
|
|
|
- collectionCards[ti.tweetId].push(cm.card);
|
|
|
- })
|
|
|
- })
|
|
|
-
|
|
|
- var collectionItems = [];
|
|
|
- Object.entries(collectionCards).forEach(function (e) {
|
|
|
- var tweetId = e[0], cards = e[1];
|
|
|
- var note = cards.map(function (card) {
|
|
|
- return card.created_at + "\n\n" + card.content;
|
|
|
- }).join("\n\n-\n\n");
|
|
|
-
|
|
|
- collectionItems.push({id: tweetId, note: note, tweet_infos: cm.tweetInfos, card_infos: cards.map(c => 'card#' + c.id)});
|
|
|
- })
|
|
|
- """
|
|
|
-
|
|
|
- collection = {
|
|
|
- 'items': [], # described in JS function above
|
|
|
- 'authorized_users': [g.twitter_user['id']]
|
|
|
- }
|
|
|
-
|
|
|
- return jsonify(collection)
|
|
|
|
|
|
#twitter_meta = TwitterMetadata('./data/meta')
|
|
|
|
|
|
@@ -567,7 +382,68 @@ def get_tweet_html (tweet_id):
|
|
|
image = user.profile_image_url
|
|
|
)
|
|
|
|
|
|
- return render_template('tweet-collection.html', user = user, tweets = tweets, query = query, page_nav=page_nav, skip_embed_replies=skip_embed_replies, opengraph_info=opengraph_info)
|
|
|
+
|
|
|
+ if view == 'replies':
|
|
|
+ tweet = tweets[0]
|
|
|
+
|
|
|
+ if tweet.id == '1608510741941989378':
|
|
|
+ unreplied = [
|
|
|
+ UnrepliedSection(
|
|
|
+ description = "Not clear what GS is still.",
|
|
|
+ span = (40, 80)
|
|
|
+ )
|
|
|
+ ]
|
|
|
+ tweet = replace(tweet,
|
|
|
+ unreplied = unreplied
|
|
|
+ )
|
|
|
+
|
|
|
+ expand_parts = request.args.get('expand')
|
|
|
+ if expand_parts:
|
|
|
+ expand_parts = expand_parts.split(',')
|
|
|
+
|
|
|
+ def reply_to_thread_item (fi):
|
|
|
+ nonlocal expand_parts
|
|
|
+
|
|
|
+ if fi.id == '1609714342211244038':
|
|
|
+ print(f'reply_to_thread_item id={fi.id}')
|
|
|
+ unreplied = [
|
|
|
+ UnrepliedSection(
|
|
|
+ description = "Is there proof of this claim?",
|
|
|
+ span = (40, 80)
|
|
|
+ )
|
|
|
+ ]
|
|
|
+ fi = replace(fi,
|
|
|
+ unreplied = unreplied
|
|
|
+ )
|
|
|
+
|
|
|
+ children = None
|
|
|
+
|
|
|
+ if expand_parts and len(expand_parts) and fi.id == expand_parts[0]:
|
|
|
+ expand_parts = expand_parts[1:]
|
|
|
+
|
|
|
+ print(f'getting expanded replied for tweet={fi.id}')
|
|
|
+
|
|
|
+ expanded_replies_response = tweet_source.get_thread(fi.id,
|
|
|
+ only_replies=True,
|
|
|
+ return_dataclass=True)
|
|
|
+ if expanded_replies_response.data:
|
|
|
+ print('we got expanded responses data')
|
|
|
+
|
|
|
+ children = list(map(lambda t: tweet_model_dc_vm(expanded_replies_response.includes, t, g.me), expanded_replies_response.data))
|
|
|
+ children = list(map(reply_to_thread_item, children))
|
|
|
+
|
|
|
+
|
|
|
+ return ThreadItem(feed_item=fi, children=children)
|
|
|
+
|
|
|
+ children = list(map(reply_to_thread_item, tweets[1:]))
|
|
|
+
|
|
|
+ root = ThreadItem(
|
|
|
+ feed_item = tweet,
|
|
|
+ children = children
|
|
|
+ )
|
|
|
+ return render_template('tweet-thread.html', user = user, root = root, query = query, page_nav=page_nav, skip_embed_replies=skip_embed_replies, opengraph_info=opengraph_info)
|
|
|
+ else:
|
|
|
+ return render_template('tweet-collection.html', user = user, tweets = tweets, query = query, page_nav=page_nav, skip_embed_replies=skip_embed_replies, opengraph_info=opengraph_info)
|
|
|
|
|
|
|
|
|
|
|
|
@@ -705,7 +581,7 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
|
|
|
'view_conversation': FeedItemAction('twitter_v2_facade.get_tweet_html', {'tweet_id': tweet.conversation_id, 'view': 'conversation'}),
|
|
|
}
|
|
|
|
|
|
- if g.twitter_user:
|
|
|
+ if g.get('twitter_user'):
|
|
|
actions.update(
|
|
|
bookmark = FeedItemAction('twitter_v2_facade.post_tweet_bookmark', {'tweet_id': tweet.id}),
|
|
|
delete_bookmark = FeedItemAction('twitter_v2_facade.delete_tweet_bookmark', {'tweet_id': tweet.id}),
|
|
|
@@ -1105,6 +981,8 @@ def get_profile_html (user_id):
|
|
|
exclude_replies = request.args.get('exclude_replies', '0')
|
|
|
exclude_retweets = request.args.get('exclude_retweets', '0')
|
|
|
|
|
|
+
|
|
|
+
|
|
|
tweet_source = ApiV2TweetSource(token)
|
|
|
response_json = tweet_source.get_user_timeline(user_id,
|
|
|
exclude_replies = exclude_replies == '1',
|
|
|
@@ -1206,6 +1084,17 @@ def get_profile_html (user_id):
|
|
|
order = 50,
|
|
|
)
|
|
|
]
|
|
|
+
|
|
|
+ if not g.twitter_user:
|
|
|
+ for uid, acct in session.items():
|
|
|
+ if uid.startswith('twitter:'):
|
|
|
+ page_nav += [
|
|
|
+ dict(
|
|
|
+ href = url_for('twitter_v2_facade.get_profile_html', user_id=user_id, me=uid),
|
|
|
+ label = f'View as {acct["id"]}',
|
|
|
+ order = 1000,
|
|
|
+ )
|
|
|
+ ]
|
|
|
|
|
|
if g.twitter_live_enabled:
|
|
|
page_nav += [
|
|
|
@@ -1227,8 +1116,25 @@ def get_profile_html (user_id):
|
|
|
]
|
|
|
|
|
|
top8 = get_top8(user_id)
|
|
|
+
|
|
|
+ brand_info = {}
|
|
|
+ if g.twitter_live_enabled:
|
|
|
+ brand = find_brand_by_account(f'twitter:{user_id}')
|
|
|
|
|
|
- return render_template('user-profile.html', user = user, tweets = tweets, query = query, opengraph_info=opengraph_info, page_nav = page_nav)
|
|
|
+ if brand:
|
|
|
+ page_nav += [
|
|
|
+ dict(
|
|
|
+ href = url_for('brands.get_brand_html', brand_id=brand['id']),
|
|
|
+ label = 'Brand Page',
|
|
|
+ order = 5000,
|
|
|
+ )
|
|
|
+ ]
|
|
|
+
|
|
|
+ brand_info = fetch_brand_info(brand)
|
|
|
+ brand_info.update({'brand': brand, 'twitter': None})
|
|
|
+
|
|
|
+ return render_template('user-profile.html', user = user, tweets = tweets, query = query, opengraph_info=opengraph_info, page_nav = page_nav, top8=top8, **brand_info)
|
|
|
+
|
|
|
|
|
|
|
|
|
|
|
|
@@ -1406,4 +1312,5 @@ def get_top8 (user_id):
|
|
|
dict(
|
|
|
id='14520320'
|
|
|
),
|
|
|
- ]
|
|
|
+ ]
|
|
|
+
|
Harlan Iverson