Compare commits
3 Commits
4f80ad381a
...
0adabd3efe
| Author | SHA1 | Date |
|---|---|---|
 Jeffrey Paul
|
0adabd3efe | |
|
Jeffrey Paul
|
bab7810cd3 | |
|
Jeffrey Paul
|
a9590c0dce |
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
|
for FN in *; do
|
||||||
|
if [[ -f "$FN" ]]; then
|
||||||
|
MTIME="$(stat -f "%Sm" "$FN")"
|
||||||
|
if [[ ! -z "$MTIME" ]]; then
|
||||||
|
# eg "Sep 13 05:02:26 2019"
|
||||||
|
YYYY="$(date -j -f "%b %d %T %Y" "$MTIME" "+%Y")"
|
||||||
|
MM="$(date -j -f "%b %d %T %Y" "$MTIME" "+%m")"
|
||||||
|
DD="$(date -j -f "%b %d %T %Y" "$MTIME" "+%d")"
|
||||||
|
TD="${YYYY}/${YYYY}-${MM}/${YYYY}-${MM}-${DD}"
|
||||||
|
if [[ ! -d ./"$TD" ]]; then
|
||||||
|
mkdir -p ./"$TD"
|
||||||
|
fi
|
||||||
|
if [[ ! -e ./"${TD}"/"$FN" ]]; then
|
||||||
|
mv ./"$FN" ./"$TD"/"$FN"
|
||||||
|
echo mv ./"$FN" ./"$TD"/"$FN"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# be advised: this script is dumb and you should use
|
||||||
|
# https://github.com/ayoisaiah/f2
|
||||||
|
# instead
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
|
for FN in *.jpg *.png *.jpeg *.heic *.gif *.GIF *.JPG *.PNG *.JPEG *.HEIC; do
|
||||||
|
if [[ -e "$FN" ]]; then
|
||||||
|
MTIME="$(stat -f "%Sm" "$FN")"
|
||||||
|
if [[ ! -z "$MTIME" ]]; then
|
||||||
|
# eg "Sep 13 05:02:26 2019"
|
||||||
|
NP="$(date -j -f "%b %d %T %Y" "$MTIME" "+%Y-%m-%dT%H%M%S")"
|
||||||
|
EXT="${FN##*.}"
|
||||||
|
LOWEREXT="$(echo "$EXT" | tr 'A-Z' 'a-z')"
|
||||||
|
TARGET="$NP.$LOWEREXT"
|
||||||
|
if [[ -e "$FN" ]]; then
|
||||||
|
if [[ ! -e "$TARGET" ]]; then
|
||||||
|
mv "$FN" "$TARGET"
|
||||||
|
echo mv "$FN" "$TARGET"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
@ -1,4 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
# be advised: this script is dumb and you should use
|
||||||
|
# https://github.com/ayoisaiah/f2
|
||||||
|
# instead
|
||||||
|
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
# this is designed to be symlinked to ~/.bashrc.d/999.goinstaller.sh
|
||||||
|
|
||||||
|
if ! which certstrap 2>&1 >/dev/null ; then
|
||||||
|
go install github.com/square/certstrap@latest
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! which f2 2>&1 >/dev/null ; then
|
||||||
|
go install github.com/ayoisaiah/f2/cmd/f2@latest
|
||||||
|
fi
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
JUNKFILES = .bash_history .irb_history .pip .ScanSnap .nbems .fldigi .cpan
|
JUNKFILES = .bash_history .irb_history .pip .ScanSnap .nbems .fldigi .cpan
|
||||||
JUNKFILES += .gdb_history .mysql_history .sqlite_history
|
JUNKFILES += .gdb_history .mysql_history .sqlite_history
|
||||||
BREWPACKAGES := mosh duplicity pv offlineimap wget nmap tor torsocks
|
BREWPACKAGES := mosh duplicity pv offlineimap wget nmap tor torsocks
|
||||||
YYYYMM := $(shell date +%Y%m)
|
YYYYMM := $(shell date +%Y-%m)
|
||||||
|
|
||||||
HACKSREPO := ~/.paths/sneak-scratch/dev/hacks/
|
HACKSREPO := ~/.paths/sneak-scratch/dev/hacks/
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
module git.eeqj.de/sneak/tweetbackup
|
||||||
|
|
||||||
|
go 1.15
|
||||||
|
|
||||||
|
require github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867 h1:X4ovGDRUmCkvcV/+C6VKo+88+5kS4fSaaegKhG0VN7U=
|
||||||
|
github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867/go.mod h1:VZADi02o1/HsNTTCPqT8kFK+RGwzVdYVdpatmzJ6Ap4=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
twitterscraper "github.com/n0madic/twitter-scraper"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
profile, err := twitterscraper.GetProfile("jack")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
prettyJSON, err := json.MarshalIndent(profile, "", " ")
|
||||||
|
fmt.Printf("%s\n", string(prettyJSON))
|
||||||
|
}
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,11 @@
|
||||||
|
PYTHON := $(shell which python)
|
||||||
|
PIPENV := $(PYTHON) -m pipenv
|
||||||
|
|
||||||
|
run:
|
||||||
|
$(PIPENV) run python3 ./main.py
|
||||||
|
|
||||||
|
shell:
|
||||||
|
$(PIPENV) shell
|
||||||
|
|
||||||
|
install:
|
||||||
|
$(PIPENV) install
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
[[source]]
|
||||||
|
name = "pypi"
|
||||||
|
url = "https://pypi.org/simple"
|
||||||
|
verify_ssl = true
|
||||||
|
|
||||||
|
[dev-packages]
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
twitterscraper = "*"
|
||||||
|
|
||||||
|
[requires]
|
||||||
|
python_version = "3.8"
|
||||||
|
|
@ -0,0 +1,140 @@
|
||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"hash": {
|
||||||
|
"sha256": "cb904de3c8c623afeb541906a8717fd6cd8968cfa5261e64bf85ab94447e013a"
|
||||||
|
},
|
||||||
|
"pipfile-spec": 6,
|
||||||
|
"requires": {
|
||||||
|
"python_version": "3.8"
|
||||||
|
},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "pypi",
|
||||||
|
"url": "https://pypi.org/simple",
|
||||||
|
"verify_ssl": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"beautifulsoup4": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||||
|
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||||
|
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||||
|
],
|
||||||
|
"version": "==4.9.3"
|
||||||
|
},
|
||||||
|
"billiard": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede",
|
||||||
|
"sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a"
|
||||||
|
],
|
||||||
|
"version": "==3.6.3.0"
|
||||||
|
},
|
||||||
|
"bs4": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
|
||||||
|
],
|
||||||
|
"version": "==0.0.1"
|
||||||
|
},
|
||||||
|
"certifi": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
|
||||||
|
"sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
|
||||||
|
],
|
||||||
|
"version": "==2020.6.20"
|
||||||
|
},
|
||||||
|
"chardet": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||||
|
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||||
|
],
|
||||||
|
"version": "==3.0.4"
|
||||||
|
},
|
||||||
|
"coala-utils": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:5f396bfe15cdbf6bcac83f0c106a8c4694a9e9a9d7c344c929b02abab604190e",
|
||||||
|
"sha256:fd7d838fbd65932c68d58c35f5eaf4508e9d92420cebceece464256d927c531c"
|
||||||
|
],
|
||||||
|
"version": "==0.5.1"
|
||||||
|
},
|
||||||
|
"idna": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||||
|
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||||
|
],
|
||||||
|
"version": "==2.10"
|
||||||
|
},
|
||||||
|
"lxml": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0e89f5d422988c65e6936e4ec0fe54d6f73f3128c80eb7ecc3b87f595523607b",
|
||||||
|
"sha256:189ad47203e846a7a4951c17694d845b6ade7917c47c64b29b86526eefc3adf5",
|
||||||
|
"sha256:1d87936cb5801c557f3e981c9c193861264c01209cb3ad0964a16310ca1b3301",
|
||||||
|
"sha256:211b3bcf5da70c2d4b84d09232534ad1d78320762e2c59dedc73bf01cb1fc45b",
|
||||||
|
"sha256:2358809cc64394617f2719147a58ae26dac9e21bae772b45cfb80baa26bfca5d",
|
||||||
|
"sha256:23c83112b4dada0b75789d73f949dbb4e8f29a0a3511647024a398ebd023347b",
|
||||||
|
"sha256:24e811118aab6abe3ce23ff0d7d38932329c513f9cef849d3ee88b0f848f2aa9",
|
||||||
|
"sha256:2d5896ddf5389560257bbe89317ca7bcb4e54a02b53a3e572e1ce4226512b51b",
|
||||||
|
"sha256:2d6571c48328be4304aee031d2d5046cbc8aed5740c654575613c5a4f5a11311",
|
||||||
|
"sha256:2e311a10f3e85250910a615fe194839a04a0f6bc4e8e5bb5cac221344e3a7891",
|
||||||
|
"sha256:302160eb6e9764168e01d8c9ec6becddeb87776e81d3fcb0d97954dd51d48e0a",
|
||||||
|
"sha256:3a7a380bfecc551cfd67d6e8ad9faa91289173bdf12e9cfafbd2bdec0d7b1ec1",
|
||||||
|
"sha256:3d9b2b72eb0dbbdb0e276403873ecfae870599c83ba22cadff2db58541e72856",
|
||||||
|
"sha256:475325e037fdf068e0c2140b818518cf6bc4aa72435c407a798b2db9f8e90810",
|
||||||
|
"sha256:4b7572145054330c8e324a72d808c8c8fbe12be33368db28c39a255ad5f7fb51",
|
||||||
|
"sha256:4fff34721b628cce9eb4538cf9a73d02e0f3da4f35a515773cce6f5fe413b360",
|
||||||
|
"sha256:56eff8c6fb7bc4bcca395fdff494c52712b7a57486e4fbde34c31bb9da4c6cc4",
|
||||||
|
"sha256:573b2f5496c7e9f4985de70b9bbb4719ffd293d5565513e04ac20e42e6e5583f",
|
||||||
|
"sha256:7ecaef52fd9b9535ae5f01a1dd2651f6608e4ec9dc136fc4dfe7ebe3c3ddb230",
|
||||||
|
"sha256:803a80d72d1f693aa448566be46ffd70882d1ad8fc689a2e22afe63035eb998a",
|
||||||
|
"sha256:8862d1c2c020cb7a03b421a9a7b4fe046a208db30994fc8ff68c627a7915987f",
|
||||||
|
"sha256:9b06690224258db5cd39a84e993882a6874676f5de582da57f3df3a82ead9174",
|
||||||
|
"sha256:a71400b90b3599eb7bf241f947932e18a066907bf84617d80817998cee81e4bf",
|
||||||
|
"sha256:bb252f802f91f59767dcc559744e91efa9df532240a502befd874b54571417bd",
|
||||||
|
"sha256:be1ebf9cc25ab5399501c9046a7dcdaa9e911802ed0e12b7d620cd4bbf0518b3",
|
||||||
|
"sha256:be7c65e34d1b50ab7093b90427cbc488260e4b3a38ef2435d65b62e9fa3d798a",
|
||||||
|
"sha256:c0dac835c1a22621ffa5e5f999d57359c790c52bbd1c687fe514ae6924f65ef5",
|
||||||
|
"sha256:c152b2e93b639d1f36ec5a8ca24cde4a8eefb2b6b83668fcd8e83a67badcb367",
|
||||||
|
"sha256:d182eada8ea0de61a45a526aa0ae4bcd222f9673424e65315c35820291ff299c",
|
||||||
|
"sha256:d18331ea905a41ae71596502bd4c9a2998902328bbabd29e3d0f5f8569fabad1",
|
||||||
|
"sha256:d20d32cbb31d731def4b1502294ca2ee99f9249b63bc80e03e67e8f8e126dea8",
|
||||||
|
"sha256:d4ad7fd3269281cb471ad6c7bafca372e69789540d16e3755dd717e9e5c9d82f",
|
||||||
|
"sha256:d6f8c23f65a4bfe4300b85f1f40f6c32569822d08901db3b6454ab785d9117cc",
|
||||||
|
"sha256:d84d741c6e35c9f3e7406cb7c4c2e08474c2a6441d59322a00dcae65aac6315d",
|
||||||
|
"sha256:e65c221b2115a91035b55a593b6eb94aa1206fa3ab374f47c6dc10d364583ff9",
|
||||||
|
"sha256:f98b6f256be6cec8dd308a8563976ddaff0bdc18b730720f6f4bee927ffe926f"
|
||||||
|
],
|
||||||
|
"version": "==4.6.1"
|
||||||
|
},
|
||||||
|
"requests": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
|
||||||
|
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
|
||||||
|
],
|
||||||
|
"version": "==2.24.0"
|
||||||
|
},
|
||||||
|
"soupsieve": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||||
|
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||||
|
],
|
||||||
|
"markers": "python_version >= '3.0'",
|
||||||
|
"version": "==2.0.1"
|
||||||
|
},
|
||||||
|
"twitterscraper": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:8cdf2e42f91469e5e8e83b81617e2d94f5f45b42cebdec123be7f986d0a7894e"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==1.6.1"
|
||||||
|
},
|
||||||
|
"urllib3": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
|
||||||
|
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
|
||||||
|
],
|
||||||
|
"version": "==1.25.11"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"develop": {}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from twitterscraper import query_tweets
|
||||||
|
|
||||||
|
def main():
|
||||||
|
list_of_tweets = query_tweets("@sneak_announce", 10)
|
||||||
|
|
||||||
|
#print the retrieved tweets to the screen:
|
||||||
|
for tweet in list_of_tweets:
|
||||||
|
print(tweet)
|
||||||
|
|
||||||
|
#Or save the retrieved tweets to file:
|
||||||
|
#file = open("output.txt","w")
|
||||||
|
#for tweet in query_tweets("Trump OR Clinton", 10):
|
||||||
|
#file.write(str(tweet.text.encode('utf-8')))
|
||||||
|
#file.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
PYTHON := $(shell which python)
|
||||||
|
PIPENV := $(PYTHON) -m pipenv
|
||||||
|
|
||||||
|
run:
|
||||||
|
$(PIPENV) run ./main.py
|
||||||
|
|
||||||
|
shell:
|
||||||
|
$(PIPENV) shell
|
||||||
|
|
||||||
|
install:
|
||||||
|
$(PIPENV) install
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
[[source]]
|
||||||
|
name = "pypi"
|
||||||
|
url = "https://pypi.org/simple"
|
||||||
|
verify_ssl = true
|
||||||
|
|
||||||
|
[dev-packages]
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
twitter-scraper = "*"
|
||||||
|
|
||||||
|
[requires]
|
||||||
|
python_version = "3.8"
|
||||||
|
|
||||||
|
[scripts]
|
||||||
|
main = "./main.py"
|
||||||
|
|
@ -0,0 +1,236 @@
|
||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"hash": {
|
||||||
|
"sha256": "65978f2dfcf8df3c4fe5e11585c8d9cf9a352f7daa8e2cc5054e49e27062106f"
|
||||||
|
},
|
||||||
|
"pipfile-spec": 6,
|
||||||
|
"requires": {
|
||||||
|
"python_version": "3.8"
|
||||||
|
},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "pypi",
|
||||||
|
"url": "https://pypi.org/simple",
|
||||||
|
"verify_ssl": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"appdirs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
|
||||||
|
"sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
|
||||||
|
],
|
||||||
|
"version": "==1.4.4"
|
||||||
|
},
|
||||||
|
"beautifulsoup4": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||||
|
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||||
|
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||||
|
],
|
||||||
|
"version": "==4.9.3"
|
||||||
|
},
|
||||||
|
"bs4": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
|
||||||
|
],
|
||||||
|
"version": "==0.0.1"
|
||||||
|
},
|
||||||
|
"certifi": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
|
||||||
|
"sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
|
||||||
|
],
|
||||||
|
"version": "==2020.6.20"
|
||||||
|
},
|
||||||
|
"chardet": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||||
|
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||||
|
],
|
||||||
|
"version": "==3.0.4"
|
||||||
|
},
|
||||||
|
"cssselect": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf",
|
||||||
|
"sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"
|
||||||
|
],
|
||||||
|
"version": "==1.1.0"
|
||||||
|
},
|
||||||
|
"fake-useragent": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
|
||||||
|
],
|
||||||
|
"version": "==0.1.11"
|
||||||
|
},
|
||||||
|
"idna": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||||
|
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||||
|
],
|
||||||
|
"version": "==2.10"
|
||||||
|
},
|
||||||
|
"lxml": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0e89f5d422988c65e6936e4ec0fe54d6f73f3128c80eb7ecc3b87f595523607b",
|
||||||
|
"sha256:189ad47203e846a7a4951c17694d845b6ade7917c47c64b29b86526eefc3adf5",
|
||||||
|
"sha256:1d87936cb5801c557f3e981c9c193861264c01209cb3ad0964a16310ca1b3301",
|
||||||
|
"sha256:211b3bcf5da70c2d4b84d09232534ad1d78320762e2c59dedc73bf01cb1fc45b",
|
||||||
|
"sha256:2358809cc64394617f2719147a58ae26dac9e21bae772b45cfb80baa26bfca5d",
|
||||||
|
"sha256:23c83112b4dada0b75789d73f949dbb4e8f29a0a3511647024a398ebd023347b",
|
||||||
|
"sha256:24e811118aab6abe3ce23ff0d7d38932329c513f9cef849d3ee88b0f848f2aa9",
|
||||||
|
"sha256:2d5896ddf5389560257bbe89317ca7bcb4e54a02b53a3e572e1ce4226512b51b",
|
||||||
|
"sha256:2d6571c48328be4304aee031d2d5046cbc8aed5740c654575613c5a4f5a11311",
|
||||||
|
"sha256:2e311a10f3e85250910a615fe194839a04a0f6bc4e8e5bb5cac221344e3a7891",
|
||||||
|
"sha256:302160eb6e9764168e01d8c9ec6becddeb87776e81d3fcb0d97954dd51d48e0a",
|
||||||
|
"sha256:3a7a380bfecc551cfd67d6e8ad9faa91289173bdf12e9cfafbd2bdec0d7b1ec1",
|
||||||
|
"sha256:3d9b2b72eb0dbbdb0e276403873ecfae870599c83ba22cadff2db58541e72856",
|
||||||
|
"sha256:475325e037fdf068e0c2140b818518cf6bc4aa72435c407a798b2db9f8e90810",
|
||||||
|
"sha256:4b7572145054330c8e324a72d808c8c8fbe12be33368db28c39a255ad5f7fb51",
|
||||||
|
"sha256:4fff34721b628cce9eb4538cf9a73d02e0f3da4f35a515773cce6f5fe413b360",
|
||||||
|
"sha256:56eff8c6fb7bc4bcca395fdff494c52712b7a57486e4fbde34c31bb9da4c6cc4",
|
||||||
|
"sha256:573b2f5496c7e9f4985de70b9bbb4719ffd293d5565513e04ac20e42e6e5583f",
|
||||||
|
"sha256:7ecaef52fd9b9535ae5f01a1dd2651f6608e4ec9dc136fc4dfe7ebe3c3ddb230",
|
||||||
|
"sha256:803a80d72d1f693aa448566be46ffd70882d1ad8fc689a2e22afe63035eb998a",
|
||||||
|
"sha256:8862d1c2c020cb7a03b421a9a7b4fe046a208db30994fc8ff68c627a7915987f",
|
||||||
|
"sha256:9b06690224258db5cd39a84e993882a6874676f5de582da57f3df3a82ead9174",
|
||||||
|
"sha256:a71400b90b3599eb7bf241f947932e18a066907bf84617d80817998cee81e4bf",
|
||||||
|
"sha256:bb252f802f91f59767dcc559744e91efa9df532240a502befd874b54571417bd",
|
||||||
|
"sha256:be1ebf9cc25ab5399501c9046a7dcdaa9e911802ed0e12b7d620cd4bbf0518b3",
|
||||||
|
"sha256:be7c65e34d1b50ab7093b90427cbc488260e4b3a38ef2435d65b62e9fa3d798a",
|
||||||
|
"sha256:c0dac835c1a22621ffa5e5f999d57359c790c52bbd1c687fe514ae6924f65ef5",
|
||||||
|
"sha256:c152b2e93b639d1f36ec5a8ca24cde4a8eefb2b6b83668fcd8e83a67badcb367",
|
||||||
|
"sha256:d182eada8ea0de61a45a526aa0ae4bcd222f9673424e65315c35820291ff299c",
|
||||||
|
"sha256:d18331ea905a41ae71596502bd4c9a2998902328bbabd29e3d0f5f8569fabad1",
|
||||||
|
"sha256:d20d32cbb31d731def4b1502294ca2ee99f9249b63bc80e03e67e8f8e126dea8",
|
||||||
|
"sha256:d4ad7fd3269281cb471ad6c7bafca372e69789540d16e3755dd717e9e5c9d82f",
|
||||||
|
"sha256:d6f8c23f65a4bfe4300b85f1f40f6c32569822d08901db3b6454ab785d9117cc",
|
||||||
|
"sha256:d84d741c6e35c9f3e7406cb7c4c2e08474c2a6441d59322a00dcae65aac6315d",
|
||||||
|
"sha256:e65c221b2115a91035b55a593b6eb94aa1206fa3ab374f47c6dc10d364583ff9",
|
||||||
|
"sha256:f98b6f256be6cec8dd308a8563976ddaff0bdc18b730720f6f4bee927ffe926f"
|
||||||
|
],
|
||||||
|
"version": "==4.6.1"
|
||||||
|
},
|
||||||
|
"mechanicalsoup": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:39a60627a97981869251837b8bd082485f2f681df8a3315836ffaa74663627bd",
|
||||||
|
"sha256:cddd80b0975f4fdafd137a473df4697854e6be3e332a5fa3200e607ee20d5238"
|
||||||
|
],
|
||||||
|
"version": "==0.12.0"
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:91666032d6723dc5905248417ef0dc9e4c51df9526aaeef271eacad6491f06a4"
|
||||||
|
],
|
||||||
|
"version": "==1.18.0"
|
||||||
|
},
|
||||||
|
"pyee": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0667fd696f55ffdf9f2646fa60557b4eeae1a427315d3938b614ee40755d18b6",
|
||||||
|
"sha256:15c6bcc14de2c2b3d8ee8923283fca182bcfab4155975a5165c66fc4bf44b680"
|
||||||
|
],
|
||||||
|
"version": "==7.0.4"
|
||||||
|
},
|
||||||
|
"pyppeteer": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:153c62666fe1d55b3941d1634733c02cafb47188fcf282987371e863d58f22e9",
|
||||||
|
"sha256:27d1f1a54e233428b4de3489d516db7d4805f67442bb77a7718cc04d7a40debb"
|
||||||
|
],
|
||||||
|
"version": "==0.2.2"
|
||||||
|
},
|
||||||
|
"pyquery": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:710eac327b87f15f74a95c3378c6ba62ef6fcfb0a6d009a7d33349c9f7e65835",
|
||||||
|
"sha256:8fcf77c72e3d602ce10a0bd4e65f57f0945c18e15627e49130c27172d4939d98"
|
||||||
|
],
|
||||||
|
"version": "==1.4.1"
|
||||||
|
},
|
||||||
|
"requests": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
|
||||||
|
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
|
||||||
|
],
|
||||||
|
"version": "==2.24.0"
|
||||||
|
},
|
||||||
|
"requests-html": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947",
|
||||||
|
"sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2"
|
||||||
|
],
|
||||||
|
"version": "==0.10.0"
|
||||||
|
},
|
||||||
|
"six": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||||
|
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||||
|
],
|
||||||
|
"version": "==1.15.0"
|
||||||
|
},
|
||||||
|
"soupsieve": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||||
|
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||||
|
],
|
||||||
|
"markers": "python_version >= '3.0'",
|
||||||
|
"version": "==2.0.1"
|
||||||
|
},
|
||||||
|
"tqdm": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
|
||||||
|
"sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
|
||||||
|
],
|
||||||
|
"version": "==4.51.0"
|
||||||
|
},
|
||||||
|
"twitter-scraper": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:03076748b01533f206f6b3e2e68c8a9b1f5ea167ff5fb4a4eb30e3000ad42f07",
|
||||||
|
"sha256:c58b6935166463f62cc3800a1c9a49212f4f948c97481d596a1d269ac349e88f"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.4.4"
|
||||||
|
},
|
||||||
|
"urllib3": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
|
||||||
|
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
|
||||||
|
],
|
||||||
|
"version": "==1.25.11"
|
||||||
|
},
|
||||||
|
"w3lib": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53",
|
||||||
|
"sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"
|
||||||
|
],
|
||||||
|
"version": "==1.22.0"
|
||||||
|
},
|
||||||
|
"websockets": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5",
|
||||||
|
"sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5",
|
||||||
|
"sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308",
|
||||||
|
"sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb",
|
||||||
|
"sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a",
|
||||||
|
"sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c",
|
||||||
|
"sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170",
|
||||||
|
"sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422",
|
||||||
|
"sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8",
|
||||||
|
"sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485",
|
||||||
|
"sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f",
|
||||||
|
"sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8",
|
||||||
|
"sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc",
|
||||||
|
"sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779",
|
||||||
|
"sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989",
|
||||||
|
"sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1",
|
||||||
|
"sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092",
|
||||||
|
"sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824",
|
||||||
|
"sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d",
|
||||||
|
"sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55",
|
||||||
|
"sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36",
|
||||||
|
"sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b"
|
||||||
|
],
|
||||||
|
"version": "==8.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"develop": {}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from twitter_scraper import get_tweets
|
||||||
|
|
||||||
|
def main():
|
||||||
|
for tweet in get_tweets('twitter', pages=1):
|
||||||
|
print(tweet['username'], ": ", tweet['text'])
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
run:
|
||||||
|
PYTHONUNBUFFERED=1 python3 twitter.py | tee -a users.csv
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,61 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import requests
|
||||||
|
import time
|
||||||
|
import shelve
|
||||||
|
|
||||||
|
# those stupid pigfuckers at twitter don't include your followers or
|
||||||
|
# following lists in your twitter data export by username, only by numeric
|
||||||
|
# userid, so you can't easily take your social graph with you, if you e.g.
|
||||||
|
# want to use nitter to RSS follow certain twitter feeds.
|
||||||
|
|
||||||
|
# additionally, the export includes your twitter user lists: but only the
|
||||||
|
# names and IDs of the lists - *none of the actual userids or usernames that
|
||||||
|
# comprise the lists*. just that you have a list of id x and name y, not
|
||||||
|
# who's actually on it.
|
||||||
|
|
||||||
|
# twitter are fucking assholes.
|
||||||
|
# i deleted my account right after this export.
|
||||||
|
# https://sneak.berlin/20201031/goodbye-twitter/
|
||||||
|
|
||||||
|
EXPORT_DATE = "2020-11-08T01:07:55Z"
|
||||||
|
|
||||||
|
# thanks random php person
|
||||||
|
CONVERTER_URL = "https://tweeterid.com/ajax.php"
|
||||||
|
|
||||||
|
def main():
|
||||||
|
followers = json.load(open('followers.json'))
|
||||||
|
followings = json.load(open('followings.json'))
|
||||||
|
|
||||||
|
with shelve.open('usernamedb') as db:
|
||||||
|
def save_username(id,username):
|
||||||
|
if username != "error":
|
||||||
|
db[str(id)] = username
|
||||||
|
print("saved db[%s]='%s'" % (id, username))
|
||||||
|
else:
|
||||||
|
raise ValueError("api error");
|
||||||
|
|
||||||
|
for following in followings:
|
||||||
|
id = following['following']['accountId']
|
||||||
|
if db.get(str(id),None) is None:
|
||||||
|
time.sleep(1)
|
||||||
|
username = requests.post(CONVERTER_URL, { "input": id }).text.strip()
|
||||||
|
save_username(id,username)
|
||||||
|
else:
|
||||||
|
username = db.get(str(id))
|
||||||
|
print("already had db[%s]=%s" % (id, username))
|
||||||
|
|
||||||
|
|
||||||
|
for follower in followers:
|
||||||
|
id = follower['follower']['accountId']
|
||||||
|
if db.get(str(id),None) is None:
|
||||||
|
time.sleep(1)
|
||||||
|
username = requests.post(CONVERTER_URL, { "input": id }).text.strip()
|
||||||
|
save_username(id,username)
|
||||||
|
else:
|
||||||
|
username = db.get(str(id))
|
||||||
|
print("already had db[%s]=%s" % (id, username))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,39 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -o nounset
|
||||||
|
set -o errexit
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
SD="/var/umbra"
|
||||||
|
|
||||||
|
function checkMounted() {
|
||||||
|
if [[ ! -e $SD/cur ]]; then
|
||||||
|
echo "umbraSync directory not mounted." > /dev/stderr
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeStats() {
|
||||||
|
checkMounted
|
||||||
|
OD="$SD/host/$(hostname -s)/stat"
|
||||||
|
if [[ ! -d "$OD" ]]; then
|
||||||
|
mkdir -p "$OD"
|
||||||
|
fi
|
||||||
|
ifconfig > "$OD/ifconfig"
|
||||||
|
netstat -in > "$OD/ifaces"
|
||||||
|
date -u "+%s" > "$OD/cur"
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorFileName () {
|
||||||
|
echo "$(date -u +"%Y-%m-%dT%H%M%SZ").$(hostname -s).$(uuidgen).error.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
function logError() {
|
||||||
|
checkMounted
|
||||||
|
OD="$SD/host/$(hostname -s)/error"
|
||||||
|
if [[ ! -d "$OD" ]]; then
|
||||||
|
mkdir -p "$OD"
|
||||||
|
fi
|
||||||
|
cat > "$OD/$(errorFileName)" <<< "$@"
|
||||||
|
}
|
||||||
|
|
||||||
Loading…
Reference in New Issue