zachwalk - Feed date scraper for Gemini (protocol)


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#!/usr/bin/env python3
"""load links from a gemini file and output the latest date for each one"""
__author__ = "Zach DeCook"
__email__ = "zachdecook@librem.one"
__copyright__ = "Copyright (C) 2021-2022 Zach DeCook"
__license__ = "AGPL"
__version__ = "3"

import sys
import socket
import ssl
import fileinput

DEFAULT = "1970-01-01"

def getnewestdate(url):
    """load the url, and find the newest date listed in a link"""
    # TODO: outsource to pre-installed cli program?
    hostname = url.split('/')[2]
    try:
     s = socket.create_connection((hostname, 1965),timeout=2)
     context = ssl.SSLContext()
     context.check_hostname = False
     context.verify_mode = ssl.CERT_NONE
     s = context.wrap_socket(s, server_hostname = hostname)
     s.sendall((url + '\r\n').encode("UTF-8"))
     fp = s.makefile("rb")
    except:
     return DEFAULT
    header = fp.readline()
    header = header.decode("UTF-8").strip()
    # TODO: something special if status is not 2x
    return gnd(fp)
def gnd(fp):
    nd = DEFAULT
    for line in fp:
     if type(line) != str:
       line=line.decode('UTF-8')
     if line.strip()[0:2] == '=>':
      desc =getdesc(line)
      date = desc[0:10]
      # basic check if it is a date
      if isDate(date) and date > nd:
        nd = date
    return nd

def replaceDateIfNewer(desc, newestdate):
    date = desc[0:10]
    if not isDate(date):
        return f'{newestdate} - {desc}'
    if newestdate > date:
        return str(newestdate) + desc[10:]
    return desc

def main():
    for line in fileinput.input(): #stdin or file from argv
        if line[0:2] == '=>':
          # don't use tabs
          url = line[2:].strip().split(' ')[0]
          if isAbsGeminiUrl(url):
            desc = getdesc(line)
            olddate = gnd(url)
            newestdate = getnewestdate(url)
            desc = replaceDateIfNewer(desc, newestdate)
            print(f'=> {url} {desc}')
            continue
        print(line.rstrip())

def isAbsGeminiUrl(url):
    return url[0:9] == 'gemini://'

def isDate(date):
    return date[0:4].isnumeric() and date[5:7].isnumeric() and date[8:].isnumeric()

def getdesc(line):
  return ' '.join(line[2:].strip().replace('	',' ').split(' ')[1:]).lstrip()

if __name__ == '__main__':
  main()