Finish PoC that messages the result over Telegram

main.py

@@ -0,0 +1,344 @@
+import os, logging
+DEBUG = os.environ.get('DEBUG')
+logging.basicConfig(
+        #filename='protogram.log',# encoding='utf-8',
+        format='[%(asctime)s] %(levelname)s %(module)s/%(funcName)s - %(message)s',
+        level=logging.DEBUG if DEBUG else logging.INFO)
+
+import asyncio
+import json
+import requests
+import time
+import pytz
+from datetime import datetime
+from bs4 import BeautifulSoup
+from telethon import TelegramClient, events
+
+import secrets
+
+TIMEZONE_CALGARY = pytz.timezone('America/Edmonton')
+
+bot = TelegramClient('data/bot', secrets.API_ID, secrets.API_HASH).start(bot_token=secrets.API_TOKEN)
+
+CHATGPT_TEMPLATE = '''Turn this forum post into an two paragraph instagram
+caption that tells about what a member of our makerspace has made. Add the
+hashtags: #makerspace #yyc #maker #diy #calgary and several relevant to the post
+at the end. Include a sentence explaining that this was made at Calgary
+Protospace, a makerspace that's non-profit and community ran. Only say the
+member's name once.  Use no more than 1000 characters. Write in third person.
+
+Title: {}
+Member: {}
+Post Body:
+```
+{}
+```'''
+
+try:
+    data = json.load(open('data/data.json'))
+except:
+    logging.info('data.json missing, initializing data.')
+    data = {}
+if 'topics' not in data: data['topics'] = {}
+if 'states' not in data: data['states'] = {}
+
+def store_data():
+    with open('data/data.json', 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+def get_sorted_category_topic_ids():
+    API_TAG_URL = 'https://forum.protospace.ca/tags/c/18/protouse-consent.json?match_all_tags=true&page={}&tags[]=protouse-consent'
+
+    topic_ids = []
+
+    try:
+        for page in range(100):
+            r = requests.get(API_TAG_URL.format(page))
+            r.raise_for_status()
+            r = r.json()
+
+            topics = r['topic_list']['topics']
+            ids = [str(t['id']) for t in topics]
+            topic_ids.extend(ids)
+
+            logging.info('Got {} topic IDs from page {}.'.format(len(ids), page))
+
+            if len(ids) == 0:
+                break
+
+    except BaseException as e:
+        logging.error('Problem getting topic IDs: {} - {}'.format(e.__class__.__name__, str(e)))
+        return False
+
+    return sorted(topic_ids, reverse=True)
+
+def get_topic_details(topic_id):
+    API_TOPIC_URL = 'https://forum.protospace.ca/t/{}.json'
+
+    try:
+        r = requests.get(API_TOPIC_URL.format(topic_id))
+        r.raise_for_status()
+        r = r.json()
+
+        logging.info('Got topic ID: {}.'.format(topic_id))
+
+    except BaseException as e:
+        logging.error('Problem getting topic details: {} - {}'.format(e.__class__.__name__, str(e)))
+        return False
+
+    return r
+
+def fetch_missing_topics(topic_ids):
+    for topic_id in topic_ids:
+        if topic_id in data['topics']:
+            continue
+
+        time.sleep(1)
+
+        topic = get_topic_details(topic_id)
+
+        if not topic:
+            continue
+
+        data['topics'][topic_id] = topic
+        store_data()
+
+        logging.info('Fetched topic {}: {}'.format(topic_id, topic['title']))
+
+def api_chatgpt(prompt):
+    thread = [
+        dict(role='system', content='You are ChatGPT, a large language model trained by OpenAI. Answer as concisely as possible. Be terse.'),
+        dict(role='user', content=prompt),
+    ]
+
+    data = dict(
+        messages=thread,
+        model='gpt-4-1106-preview',
+        temperature=0.5,
+        user='protogram',
+        max_tokens=1000,
+    )
+    headers = {'Authorization': 'Bearer ' + secrets.OPENAI_KEY}
+
+    start = time.time()
+
+    try:
+        r = requests.post('https://api.openai.com/v1/chat/completions', json=data, headers=headers, timeout=40)
+        r.raise_for_status()
+        r = r.json()
+
+        gpt_reply = r['choices'][0]['message']['content']
+    except BaseException as e:
+        logging.error('Problem with chatgpt: {} - {}'.format(e.__class__.__name__, str(e)))
+        return False
+
+    end = time.time()
+
+    logging.info('Got ChatGPT response in {}s:\n{}'.format(str(end - start), gpt_reply))
+    return gpt_reply
+
+def get_portal_name_from_discourse(username):
+    try:
+        params = dict(discourse_username=username)
+        headers = {'Authorization': 'Bearer ' + secrets.PROTOGRAM_API_KEY}
+        r = requests.get('https://api.my.protospace.ca/search/discourse/', params=params, headers=headers, timeout=5)
+        r.raise_for_status()
+        r = r.json()
+
+        return r['member']['preferred_name']
+    except BaseException as e:
+        logging.error('Problem with getting member name: {} - {}'.format(e.__class__.__name__, str(e)))
+        return False
+
+def generate_caption(topic):
+    title = topic['title']
+
+    username = topic['post_stream']['posts'][0]['username']
+    member = get_portal_name_from_discourse(username)
+    if not member:
+        return False
+    logging.info('Converted discourse username {} -> {}'.format(username, member))
+
+    post_html = topic['post_stream']['posts'][0]['cooked']
+
+    soup = BeautifulSoup(post_html, 'html.parser')
+    lines = soup.get_text().split('\n')
+    filtered_lines = [line for line in lines if 'KB' not in line and 'MB' not in line]
+    body = '\n'.join(filtered_lines).replace('\n\n\n', '\n\n')
+
+    prompt = CHATGPT_TEMPLATE.format(title, member, body)
+
+    logging.info('Generating caption for: {}\n{}'.format(title, body))
+
+    return api_chatgpt(prompt)
+
+def test_generate_topic_ids(topic_ids):
+    logging.info('Test generating topic IDs {} with template:\n{}\n\n'.format(str(topic_ids), CHATGPT_TEMPLATE))
+
+    for topic_id in topic_ids:
+        generate_caption(data['topics'][topic_id])
+        logging.info('Finished topic ID {}\n\n\n'.format(topic_id))
+
+    logging.info('Done.')
+
+def save_images_from_topic(topic_id, topic, state):
+    link_counts = topic['post_stream']['posts'][0]['link_counts']
+
+    urls = [x['url'] for x in link_counts if x['url'].endswith('.jpeg')]  # skip png for now
+
+    if len(urls) == 0:
+        logging.info('No photos found.')
+        return False
+
+    logging.info('Downloading {} photos...'.format(len(urls)))
+
+    count = 0
+    for url in urls:
+        logging.info('Downloading photo: {}'.format(url))
+
+        try:
+            r = requests.get(url, timeout=10)
+            r.raise_for_status()
+            filename = 'data/photos/{}_{}.jpg'.format(topic_id, count)
+
+            with open(filename, 'wb') as f:
+                f.write(r.content)
+
+            state['photos'].append(filename)
+            store_data()
+
+            count += 1
+        except BaseException as e:
+            logging.error('Problem downloading photo: {} - {}'.format(e.__class__.__name__, str(e)))
+            continue
+
+    return count
+
+def find_next_valid_topic_id(topic_ids):
+    for topic_id in topic_ids:
+        try:
+            if data['states'][topic_id]['status'] in ['POSTED', 'ERROR']:
+                continue
+        except KeyError:
+            break
+    else:  # for loop
+        return False
+
+    return topic_id
+
+async def send_data_to_admin(state):
+    try:
+        await bot.send_message(
+           secrets.ADMIN_TELEGRAM_ID,
+           state['caption'],
+           file=state['photos'],
+        )
+        return True
+    except BaseException as e:
+        logging.error('Problem sending to admin: {} - {}'.format(e.__class__.__name__, str(e)))
+        return False
+
+
+async def process_topics():
+    while True:
+        #await asyncio.sleep(60)
+
+        FRIDAY = 4
+        now = datetime.now(TIMEZONE_CALGARY)
+        
+        #if not (now.weekday() == FRIDAY and now.hour == 18 and now.minute == 15):
+        #    continue
+
+        logging.info('Processing topics...')
+
+        topic_ids = get_sorted_category_topic_ids()
+        fetch_missing_topics(topic_ids)
+
+        topic_id = find_next_valid_topic_id(topic_ids)
+
+        if not topic_id:
+            logging.info('No next valid topic ID found.')
+            continue
+
+        logging.info('Next valid topic ID: {}'.format(topic_id))
+
+        if topic_id not in data['states']:
+            data['states'][topic_id] = dict(
+                status='NEW',
+                photos=[],
+                caption=None
+            )
+
+        topic = data['topics'][topic_id]
+        state = data['states'][topic_id]
+
+        count = save_images_from_topic(topic_id, topic, state)
+
+        if not count:
+            state['status'] = 'ERROR'
+            store_data()
+            continue
+
+        caption = generate_caption(topic)
+        state['caption'] = caption
+
+        result = await send_data_to_admin(state)
+
+        if result:
+            state['status'] = 'POSTED'
+        else:
+            state['status'] = 'ERROR'
+        store_data()
+
+        print('done')
+        while True: pass
+
+
+
+@bot.on(events.NewMessage(pattern='/start'))
+async def start(event):
+    await event.respond('Hello world')
+    raise events.StopPropagation
+
+@bot.on(events.NewMessage)
+async def new_message(event):
+    if not event.raw_text:
+        logging.info('No text found')
+        return
+
+    logging.info('Message: ' + event.raw_text)
+
+    if event.sender.id != secrets.ADMIN_TELEGRAM_ID:
+        logging.info('Message not from Admin')
+        return
+
+    await event.respond(event.text)
+
+
+def task_died(future):
+    if os.environ.get('SHELL'):
+        logging.error('Protogram task died!')
+    else:
+        logging.error('Protogram task died! Waiting 60s and exiting...')
+        try:
+            #controller_message('Protogram task died! Waiting 60s and exiting...')
+            pass
+        except:  # we want this to succeed no matter what
+            pass
+        time.sleep(60)
+    exit()
+
+if __name__ == '__main__':
+    logging.info('===== BOOT UP =====')
+
+    #topic_ids = get_sorted_category_topic_ids()
+    #print(topic_ids)
+    #fetch_missing_topics(topic_ids)
+    #print('next valid:', find_next_valid_topic_id(topic_ids))
+    #generate_caption(data['topics']['5363'])
+    #test_generate_topic_ids(['5174', '5381', '5363', '5205', '5273'])
+
+    loop = asyncio.get_event_loop()
+    a = loop.create_task(process_topics()).add_done_callback(task_died)
+    loop.run_forever()