Added a new link caching system, updated the readme, started ignoreing pycache
This commit is contained in:
Robin Universe
parent
52df884b0f
commit
ecba0fc79d
|
|
@ -0,0 +1 @@
|
|||
__pycache__/
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
13
readme.md
13
readme.md
|
|
@ -1,6 +1,6 @@
|
|||
# TwitFix
|
||||
|
||||
very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page
|
||||
very basic flask server that fixes twitter embeds in discord by using youtube-dl to grab the direct link to the MP4 file and embeds the link to it in a custom page, with link caching
|
||||
|
||||
This does work! but I'm new to flask, so it can probably be improved a great deal.
|
||||
|
||||
|
|
@ -9,16 +9,23 @@ This does work! but I'm new to flask, so it can probably be improved a great dea
|
|||
just put the url to the server, and directly after, the full URL to the tweet you want to embed
|
||||
|
||||
**I now have a copy of this running on a Linode server, you can use it via the following url**
|
||||
`http://twtfx.me/<twitter url>`
|
||||
|
||||
https://fxtwitter.com/<twitter video url>
|
||||
|
||||
You can also simply type out 'fx' directly before 'twitter.com' in any valid twitter video url, and that will convert it into a working TwitFix url, for example:
|
||||
|
||||
|
||||
|
||||
**Note**: If you enjoy this service, please considering donating via [Ko-Fi](https://ko-fi.com/robin_universe) to help cover server costs
|
||||
|
||||
## How to run (server side)
|
||||
|
||||
this script uses the youtube-dl python module, along with flask, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 )
|
||||
this script uses the youtube-dl python module, along with flask and pymongo, so install those with pip (you can use `pip install -r requirements.txt`) and start the server with `python twitfix.py` ( will need sudo if you leave it at port 80 )
|
||||
|
||||
By default I have the port set to 80, just cause that's what was convenient for me, but it can easily be changed, either using an environment variable, or changing the bottom line of the script itself
|
||||
|
||||
I have included some files to give you a head start on setting this server up with uWSGI, though if you decide to use uWSGI I suggest you set up mongoDB link caching by going into the script and change `link_cache_system` from `json` to `"DB"`, and inserting you mongoDB address, as having many workers writing to the same json file doesn't really work
|
||||
|
||||
This project is licensed under the **Do What The Fuck You Want Public License**
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
Flask
|
||||
youtube_dl
|
||||
pymongo
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
[uwsgi]
|
||||
module = wsgi:app
|
||||
|
||||
master = true
|
||||
processes = 5
|
||||
|
||||
socket = 80
|
||||
chmod-socket = 660
|
||||
vacuum = true
|
||||
|
||||
die-on-term = true
|
||||
92
twitfix.py
92
twitfix.py
|
|
@ -1,4 +1,5 @@
|
|||
from flask import Flask, render_template
|
||||
import pymongo
|
||||
import youtube_dl
|
||||
import json
|
||||
import re
|
||||
|
|
@ -6,14 +7,20 @@ import re
|
|||
app = Flask(__name__)
|
||||
pathregex = re.compile("\\w{1,15}\\/status\\/\\d{19}")
|
||||
|
||||
link_cache = {}
|
||||
f = open('links.json',)
|
||||
link_cache = json.load(f)
|
||||
f.close()
|
||||
link_cache_system = "json" # Select your prefered link cache system, "db" for mongoDB or "json" for locally writing a json file ( not great if using multiple workers )
|
||||
|
||||
if link_cache_system == "json":
|
||||
link_cache = {}
|
||||
f = open('links.json',)
|
||||
link_cache = json.load(f)
|
||||
f.close()
|
||||
elif link_cache_system == "db":
|
||||
client = pymongo.MongoClient("PUT YOUR MONGODB URL HERE")
|
||||
db = client.TwitFix
|
||||
|
||||
@app.route('/')
|
||||
def default():
|
||||
return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord click this link to be directed to my github page for the project!")
|
||||
return render_template('default.html', message="TwitFix is an attempt to fix twitter video embeds in discord! created by Robin Universe :) 💖 ")
|
||||
|
||||
@app.route('/<path:subpath>')
|
||||
def twitfix(subpath):
|
||||
|
|
@ -43,28 +50,65 @@ def info(subpath):
|
|||
return result
|
||||
|
||||
def embedVideo(vidlink): # Return a render template from a video url
|
||||
if vidlink in link_cache:
|
||||
print("Link located in cache")
|
||||
return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink)
|
||||
if link_cache_system == "db":
|
||||
collection = db.linkCache
|
||||
dbresult = collection.find_one({'tweet': vidlink})
|
||||
if dbresult != None:
|
||||
print("Link located in DB cache")
|
||||
return render_template('index.html', vidurl=dbresult['url'], desc=dbresult['description'], pic=dbresult['thumbnail'], user=dbresult['uploader'], vidlink=vidlink)
|
||||
else:
|
||||
with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
|
||||
try:
|
||||
print("Link not in json cache, downloading and adding details to cache file")
|
||||
result = ydl.extract_info(vidlink, download=False)
|
||||
vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
|
||||
|
||||
try:
|
||||
out = db.linkCache.insert_one(vnf)
|
||||
print("Link added to DB cache")
|
||||
except Exception:
|
||||
print("Failed to add link to DB cache")
|
||||
|
||||
return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
|
||||
|
||||
except Exception:
|
||||
print("Failed to download link")
|
||||
return render_template('default.html', message="Failed to scan your link!")
|
||||
|
||||
elif link_cache_system == "json":
|
||||
if vidlink in link_cache:
|
||||
print("Link located in json cache")
|
||||
return render_template('index.html', vidurl=link_cache[vidlink]['url'], desc=link_cache[vidlink]['description'], pic=link_cache[vidlink]['thumbnail'], user=link_cache[vidlink]['uploader'], vidlink=vidlink)
|
||||
else:
|
||||
with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
|
||||
try:
|
||||
print("Link not in json cache, downloading and adding details to cache file")
|
||||
result = ydl.extract_info(vidlink, download=False)
|
||||
vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
|
||||
link_cache[vidlink] = vnf
|
||||
|
||||
with open("links.json", "w") as outfile:
|
||||
json.dump(link_cache, outfile, indent=4, sort_keys=True)
|
||||
|
||||
except Exception: # Just to keep from 500s that are messy
|
||||
print("Failed to download link")
|
||||
return render_template('default.html', message="Failed to scan your link!")
|
||||
|
||||
return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
|
||||
else:
|
||||
with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
|
||||
try:
|
||||
print("Link not in cache, downloading and adding details to cache file")
|
||||
result = ydl.extract_info(vidlink, download=False)
|
||||
vnf = vidInfo(result['url'], result['description'], result['thumbnail'], result['uploader'])
|
||||
link_cache[vidlink] = vnf
|
||||
try:
|
||||
with youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'}) as ydl:
|
||||
result = ydl.extract_info(subpath, download=False)
|
||||
vnf = vidInfo(result['url'], vidlink, result['description'], result['thumbnail'], result['uploader'])
|
||||
return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
|
||||
except Exception:
|
||||
print("Failed to download link")
|
||||
return render_template('default.html', message="Failed to scan your link!")
|
||||
|
||||
with open("links.json", "w") as outfile:
|
||||
json.dump(link_cache, outfile, indent=4, sort_keys=True)
|
||||
|
||||
except Exception: # Just to keep from 500s that are messy
|
||||
print("Failed to download link")
|
||||
return render_template('default.html', message="Failed to scan your link!")
|
||||
|
||||
return render_template('index.html', vidurl=vnf['url'], desc=vnf['description'], pic=vnf['thumbnail'], user=vnf['uploader'], vidlink=vidlink)
|
||||
|
||||
def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info with default values
|
||||
def vidInfo(url, tweet="", desc="", thumb="", uploader=""): # Return a dict of video info with default values
|
||||
vnf = {
|
||||
"tweet" :tweet,
|
||||
"url" :url,
|
||||
"description" :desc,
|
||||
"thumbnail" :thumb,
|
||||
|
|
@ -73,4 +117,4 @@ def vidInfo(url, desc="", thumb="", uploader=""): # Return a dict of video info
|
|||
return vnf
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host='0.0.0.0', port=80)
|
||||
app.run(host='0.0.0.0')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Init file for twitfix uwsgi instance
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=robin
|
||||
Group=robin
|
||||
WorkingDirectory=/home/robin/twitfix
|
||||
Environment="PATH=/home/robin/twitfix/twitfixenv/bin"
|
||||
ExecStart=/home/robin/twitfix/twitfixenv/bin/uwsgi --ini twitfix.ini
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue