master
Jeffrey Paul 2 years ago
parent a9590c0dce
commit bab7810cd3
  1. 23
      bin/file-by-date
  2. 22
      bin/rename-images-by-mtime
  3. 2
      homedir.makefile/Makefile
  4. 5
      tweetbackup/go/go.mod
  5. 14
      tweetbackup/go/go.sum
  6. 17
      tweetbackup/go/main.go
  7. BIN
      tweetbackup/go/tweetbackup
  8. 11
      tweetbackup/python.2/Makefile
  9. 12
      tweetbackup/python.2/Pipfile
  10. 140
      tweetbackup/python.2/Pipfile.lock
  11. 19
      tweetbackup/python.2/main.py
  12. 11
      tweetbackup/python/Makefile
  13. 15
      tweetbackup/python/Pipfile
  14. 236
      tweetbackup/python/Pipfile.lock
  15. 9
      tweetbackup/python/main.py
  16. 2
      twitter-goodbye/Makefile
  17. 53636
      twitter-goodbye/followers.json
  18. 55808
      twitter-goodbye/followings.json
  19. 61
      twitter-goodbye/twitter.py
  20. BIN
      twitter-goodbye/usernamedb.db
  21. 4525
      twitter-goodbye/users.csv
  22. 39
      umbrasync/umbraStats.sh

@ -0,0 +1,23 @@
#!/bin/bash
set -x
for FN in *; do
if [[ -f "$FN" ]]; then
MTIME="$(stat -f "%Sm" "$FN")"
if [[ ! -z "$MTIME" ]]; then
# eg "Sep 13 05:02:26 2019"
YYYY="$(date -j -f "%b %d %T %Y" "$MTIME" "+%Y")"
MM="$(date -j -f "%b %d %T %Y" "$MTIME" "+%m")"
DD="$(date -j -f "%b %d %T %Y" "$MTIME" "+%d")"
TD="${YYYY}/${YYYY}-${MM}/${YYYY}-${MM}-${DD}"
if [[ ! -d ./"$TD" ]]; then
mkdir -p ./"$TD"
fi
if [[ ! -e ./"${TD}"/"$FN" ]]; then
mv ./"$FN" ./"$TD"/"$FN"
echo mv ./"$FN" ./"$TD"/"$FN"
fi
fi
fi
done

@ -0,0 +1,22 @@
#!/bin/bash
set -x
for FN in *.jpg *.png *.jpeg *.heic *.gif *.GIF *.JPG *.PNG *.JPEG *.HEIC; do
if [[ -e "$FN" ]]; then
MTIME="$(stat -f "%Sm" "$FN")"
if [[ ! -z "$MTIME" ]]; then
# eg "Sep 13 05:02:26 2019"
NP="$(date -j -f "%b %d %T %Y" "$MTIME" "+%Y-%m-%dT%H%M%S")"
EXT="${FN##*.}"
LOWEREXT="$(echo "$EXT" | tr 'A-Z' 'a-z')"
TARGET="$NP.$LOWEREXT"
if [[ -e "$FN" ]]; then
if [[ ! -e "$TARGET" ]]; then
mv "$FN" "$TARGET"
echo mv "$FN" "$TARGET"
fi
fi
fi
fi
done

@ -1,7 +1,7 @@
JUNKFILES = .bash_history .irb_history .pip .ScanSnap .nbems .fldigi .cpan JUNKFILES = .bash_history .irb_history .pip .ScanSnap .nbems .fldigi .cpan
JUNKFILES += .gdb_history .mysql_history .sqlite_history JUNKFILES += .gdb_history .mysql_history .sqlite_history
BREWPACKAGES := mosh duplicity pv offlineimap wget nmap tor torsocks BREWPACKAGES := mosh duplicity pv offlineimap wget nmap tor torsocks
YYYYMM := $(shell date +%Y%m) YYYYMM := $(shell date +%Y-%m)
HACKSREPO := ~/.paths/sneak-scratch/dev/hacks/ HACKSREPO := ~/.paths/sneak-scratch/dev/hacks/

@ -0,0 +1,5 @@
module git.eeqj.de/sneak/tweetbackup
go 1.15
require github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867

@ -0,0 +1,14 @@
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867 h1:X4ovGDRUmCkvcV/+C6VKo+88+5kS4fSaaegKhG0VN7U=
github.com/n0madic/twitter-scraper v0.0.0-20201012064956-48bfdb661867/go.mod h1:VZADi02o1/HsNTTCPqT8kFK+RGwzVdYVdpatmzJ6Ap4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

@ -0,0 +1,17 @@
package main
import (
"encoding/json"
"fmt"
twitterscraper "github.com/n0madic/twitter-scraper"
)
func main() {
profile, err := twitterscraper.GetProfile("jack")
if err != nil {
panic(err)
}
prettyJSON, err := json.MarshalIndent(profile, "", " ")
fmt.Printf("%s\n", string(prettyJSON))
}

Binary file not shown.

@ -0,0 +1,11 @@
PYTHON := $(shell which python)
PIPENV := $(PYTHON) -m pipenv
run:
$(PIPENV) run python3 ./main.py
shell:
$(PIPENV) shell
install:
$(PIPENV) install

@ -0,0 +1,12 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
twitterscraper = "*"
[requires]
python_version = "3.8"

@ -0,0 +1,140 @@
{
"_meta": {
"hash": {
"sha256": "cb904de3c8c623afeb541906a8717fd6cd8968cfa5261e64bf85ab94447e013a"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.8"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"beautifulsoup4": {
"hashes": [
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
],
"version": "==4.9.3"
},
"billiard": {
"hashes": [
"sha256:bff575450859a6e0fbc2f9877d9b715b0bbc07c3565bb7ed2280526a0cdf5ede",
"sha256:d91725ce6425f33a97dfa72fb6bfef0e47d4652acd98a032bd1a7fbf06d5fa6a"
],
"version": "==3.6.3.0"
},
"bs4": {
"hashes": [
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
],
"version": "==0.0.1"
},
"certifi": {
"hashes": [
"sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
"sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
],
"version": "==2020.6.20"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"coala-utils": {
"hashes": [
"sha256:5f396bfe15cdbf6bcac83f0c106a8c4694a9e9a9d7c344c929b02abab604190e",
"sha256:fd7d838fbd65932c68d58c35f5eaf4508e9d92420cebceece464256d927c531c"
],
"version": "==0.5.1"
},
"idna": {
"hashes": [
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
],
"version": "==2.10"
},
"lxml": {
"hashes": [
"sha256:0e89f5d422988c65e6936e4ec0fe54d6f73f3128c80eb7ecc3b87f595523607b",
"sha256:189ad47203e846a7a4951c17694d845b6ade7917c47c64b29b86526eefc3adf5",
"sha256:1d87936cb5801c557f3e981c9c193861264c01209cb3ad0964a16310ca1b3301",
"sha256:211b3bcf5da70c2d4b84d09232534ad1d78320762e2c59dedc73bf01cb1fc45b",
"sha256:2358809cc64394617f2719147a58ae26dac9e21bae772b45cfb80baa26bfca5d",
"sha256:23c83112b4dada0b75789d73f949dbb4e8f29a0a3511647024a398ebd023347b",
"sha256:24e811118aab6abe3ce23ff0d7d38932329c513f9cef849d3ee88b0f848f2aa9",
"sha256:2d5896ddf5389560257bbe89317ca7bcb4e54a02b53a3e572e1ce4226512b51b",
"sha256:2d6571c48328be4304aee031d2d5046cbc8aed5740c654575613c5a4f5a11311",
"sha256:2e311a10f3e85250910a615fe194839a04a0f6bc4e8e5bb5cac221344e3a7891",
"sha256:302160eb6e9764168e01d8c9ec6becddeb87776e81d3fcb0d97954dd51d48e0a",
"sha256:3a7a380bfecc551cfd67d6e8ad9faa91289173bdf12e9cfafbd2bdec0d7b1ec1",
"sha256:3d9b2b72eb0dbbdb0e276403873ecfae870599c83ba22cadff2db58541e72856",
"sha256:475325e037fdf068e0c2140b818518cf6bc4aa72435c407a798b2db9f8e90810",
"sha256:4b7572145054330c8e324a72d808c8c8fbe12be33368db28c39a255ad5f7fb51",
"sha256:4fff34721b628cce9eb4538cf9a73d02e0f3da4f35a515773cce6f5fe413b360",
"sha256:56eff8c6fb7bc4bcca395fdff494c52712b7a57486e4fbde34c31bb9da4c6cc4",
"sha256:573b2f5496c7e9f4985de70b9bbb4719ffd293d5565513e04ac20e42e6e5583f",
"sha256:7ecaef52fd9b9535ae5f01a1dd2651f6608e4ec9dc136fc4dfe7ebe3c3ddb230",
"sha256:803a80d72d1f693aa448566be46ffd70882d1ad8fc689a2e22afe63035eb998a",
"sha256:8862d1c2c020cb7a03b421a9a7b4fe046a208db30994fc8ff68c627a7915987f",
"sha256:9b06690224258db5cd39a84e993882a6874676f5de582da57f3df3a82ead9174",
"sha256:a71400b90b3599eb7bf241f947932e18a066907bf84617d80817998cee81e4bf",
"sha256:bb252f802f91f59767dcc559744e91efa9df532240a502befd874b54571417bd",
"sha256:be1ebf9cc25ab5399501c9046a7dcdaa9e911802ed0e12b7d620cd4bbf0518b3",
"sha256:be7c65e34d1b50ab7093b90427cbc488260e4b3a38ef2435d65b62e9fa3d798a",
"sha256:c0dac835c1a22621ffa5e5f999d57359c790c52bbd1c687fe514ae6924f65ef5",
"sha256:c152b2e93b639d1f36ec5a8ca24cde4a8eefb2b6b83668fcd8e83a67badcb367",
"sha256:d182eada8ea0de61a45a526aa0ae4bcd222f9673424e65315c35820291ff299c",
"sha256:d18331ea905a41ae71596502bd4c9a2998902328bbabd29e3d0f5f8569fabad1",
"sha256:d20d32cbb31d731def4b1502294ca2ee99f9249b63bc80e03e67e8f8e126dea8",
"sha256:d4ad7fd3269281cb471ad6c7bafca372e69789540d16e3755dd717e9e5c9d82f",
"sha256:d6f8c23f65a4bfe4300b85f1f40f6c32569822d08901db3b6454ab785d9117cc",
"sha256:d84d741c6e35c9f3e7406cb7c4c2e08474c2a6441d59322a00dcae65aac6315d",
"sha256:e65c221b2115a91035b55a593b6eb94aa1206fa3ab374f47c6dc10d364583ff9",
"sha256:f98b6f256be6cec8dd308a8563976ddaff0bdc18b730720f6f4bee927ffe926f"
],
"version": "==4.6.1"
},
"requests": {
"hashes": [
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
],
"version": "==2.24.0"
},
"soupsieve": {
"hashes": [
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
],
"markers": "python_version >= '3.0'",
"version": "==2.0.1"
},
"twitterscraper": {
"hashes": [
"sha256:8cdf2e42f91469e5e8e83b81617e2d94f5f45b42cebdec123be7f986d0a7894e"
],
"index": "pypi",
"version": "==1.6.1"
},
"urllib3": {
"hashes": [
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
],
"version": "==1.25.11"
}
},
"develop": {}
}

@ -0,0 +1,19 @@
#!/usr/bin/env python3
from twitterscraper import query_tweets
def main():
list_of_tweets = query_tweets("@sneak_announce", 10)
#print the retrieved tweets to the screen:
for tweet in list_of_tweets:
print(tweet)
#Or save the retrieved tweets to file:
#file = open("output.txt","w")
#for tweet in query_tweets("Trump OR Clinton", 10):
#file.write(str(tweet.text.encode('utf-8')))
#file.close()
if __name__ == "__main__":
main()

@ -0,0 +1,11 @@
PYTHON := $(shell which python)
PIPENV := $(PYTHON) -m pipenv
run:
$(PIPENV) run ./main.py
shell:
$(PIPENV) shell
install:
$(PIPENV) install

@ -0,0 +1,15 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
twitter-scraper = "*"
[requires]
python_version = "3.8"
[scripts]
main = "./main.py"

@ -0,0 +1,236 @@
{
"_meta": {
"hash": {
"sha256": "65978f2dfcf8df3c4fe5e11585c8d9cf9a352f7daa8e2cc5054e49e27062106f"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.8"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"appdirs": {
"hashes": [
"sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
"sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
],
"version": "==1.4.4"
},
"beautifulsoup4": {
"hashes": [
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
],
"version": "==4.9.3"
},
"bs4": {
"hashes": [
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
],
"version": "==0.0.1"
},
"certifi": {
"hashes": [
"sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
"sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
],
"version": "==2020.6.20"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"cssselect": {
"hashes": [
"sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf",
"sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"
],
"version": "==1.1.0"
},
"fake-useragent": {
"hashes": [
"sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
],
"version": "==0.1.11"
},
"idna": {
"hashes": [
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
],
"version": "==2.10"
},
"lxml": {
"hashes": [
"sha256:0e89f5d422988c65e6936e4ec0fe54d6f73f3128c80eb7ecc3b87f595523607b",
"sha256:189ad47203e846a7a4951c17694d845b6ade7917c47c64b29b86526eefc3adf5",
"sha256:1d87936cb5801c557f3e981c9c193861264c01209cb3ad0964a16310ca1b3301",
"sha256:211b3bcf5da70c2d4b84d09232534ad1d78320762e2c59dedc73bf01cb1fc45b",
"sha256:2358809cc64394617f2719147a58ae26dac9e21bae772b45cfb80baa26bfca5d",
"sha256:23c83112b4dada0b75789d73f949dbb4e8f29a0a3511647024a398ebd023347b",
"sha256:24e811118aab6abe3ce23ff0d7d38932329c513f9cef849d3ee88b0f848f2aa9",
"sha256:2d5896ddf5389560257bbe89317ca7bcb4e54a02b53a3e572e1ce4226512b51b",
"sha256:2d6571c48328be4304aee031d2d5046cbc8aed5740c654575613c5a4f5a11311",
"sha256:2e311a10f3e85250910a615fe194839a04a0f6bc4e8e5bb5cac221344e3a7891",
"sha256:302160eb6e9764168e01d8c9ec6becddeb87776e81d3fcb0d97954dd51d48e0a",
"sha256:3a7a380bfecc551cfd67d6e8ad9faa91289173bdf12e9cfafbd2bdec0d7b1ec1",
"sha256:3d9b2b72eb0dbbdb0e276403873ecfae870599c83ba22cadff2db58541e72856",
"sha256:475325e037fdf068e0c2140b818518cf6bc4aa72435c407a798b2db9f8e90810",
"sha256:4b7572145054330c8e324a72d808c8c8fbe12be33368db28c39a255ad5f7fb51",
"sha256:4fff34721b628cce9eb4538cf9a73d02e0f3da4f35a515773cce6f5fe413b360",
"sha256:56eff8c6fb7bc4bcca395fdff494c52712b7a57486e4fbde34c31bb9da4c6cc4",
"sha256:573b2f5496c7e9f4985de70b9bbb4719ffd293d5565513e04ac20e42e6e5583f",
"sha256:7ecaef52fd9b9535ae5f01a1dd2651f6608e4ec9dc136fc4dfe7ebe3c3ddb230",
"sha256:803a80d72d1f693aa448566be46ffd70882d1ad8fc689a2e22afe63035eb998a",
"sha256:8862d1c2c020cb7a03b421a9a7b4fe046a208db30994fc8ff68c627a7915987f",
"sha256:9b06690224258db5cd39a84e993882a6874676f5de582da57f3df3a82ead9174",
"sha256:a71400b90b3599eb7bf241f947932e18a066907bf84617d80817998cee81e4bf",
"sha256:bb252f802f91f59767dcc559744e91efa9df532240a502befd874b54571417bd",
"sha256:be1ebf9cc25ab5399501c9046a7dcdaa9e911802ed0e12b7d620cd4bbf0518b3",
"sha256:be7c65e34d1b50ab7093b90427cbc488260e4b3a38ef2435d65b62e9fa3d798a",
"sha256:c0dac835c1a22621ffa5e5f999d57359c790c52bbd1c687fe514ae6924f65ef5",
"sha256:c152b2e93b639d1f36ec5a8ca24cde4a8eefb2b6b83668fcd8e83a67badcb367",
"sha256:d182eada8ea0de61a45a526aa0ae4bcd222f9673424e65315c35820291ff299c",
"sha256:d18331ea905a41ae71596502bd4c9a2998902328bbabd29e3d0f5f8569fabad1",
"sha256:d20d32cbb31d731def4b1502294ca2ee99f9249b63bc80e03e67e8f8e126dea8",
"sha256:d4ad7fd3269281cb471ad6c7bafca372e69789540d16e3755dd717e9e5c9d82f",
"sha256:d6f8c23f65a4bfe4300b85f1f40f6c32569822d08901db3b6454ab785d9117cc",
"sha256:d84d741c6e35c9f3e7406cb7c4c2e08474c2a6441d59322a00dcae65aac6315d",
"sha256:e65c221b2115a91035b55a593b6eb94aa1206fa3ab374f47c6dc10d364583ff9",
"sha256:f98b6f256be6cec8dd308a8563976ddaff0bdc18b730720f6f4bee927ffe926f"
],
"version": "==4.6.1"
},
"mechanicalsoup": {
"hashes": [
"sha256:39a60627a97981869251837b8bd082485f2f681df8a3315836ffaa74663627bd",
"sha256:cddd80b0975f4fdafd137a473df4697854e6be3e332a5fa3200e607ee20d5238"
],
"version": "==0.12.0"
},
"parse": {
"hashes": [
"sha256:91666032d6723dc5905248417ef0dc9e4c51df9526aaeef271eacad6491f06a4"
],
"version": "==1.18.0"
},
"pyee": {
"hashes": [
"sha256:0667fd696f55ffdf9f2646fa60557b4eeae1a427315d3938b614ee40755d18b6",
"sha256:15c6bcc14de2c2b3d8ee8923283fca182bcfab4155975a5165c66fc4bf44b680"
],
"version": "==7.0.4"
},
"pyppeteer": {
"hashes": [
"sha256:153c62666fe1d55b3941d1634733c02cafb47188fcf282987371e863d58f22e9",
"sha256:27d1f1a54e233428b4de3489d516db7d4805f67442bb77a7718cc04d7a40debb"
],
"version": "==0.2.2"
},
"pyquery": {
"hashes": [
"sha256:710eac327b87f15f74a95c3378c6ba62ef6fcfb0a6d009a7d33349c9f7e65835",
"sha256:8fcf77c72e3d602ce10a0bd4e65f57f0945c18e15627e49130c27172d4939d98"
],
"version": "==1.4.1"
},
"requests": {
"hashes": [
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
],
"version": "==2.24.0"
},
"requests-html": {
"hashes": [
"sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947",
"sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2"
],
"version": "==0.10.0"
},
"six": {
"hashes": [
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
],
"version": "==1.15.0"
},
"soupsieve": {
"hashes": [
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
],
"markers": "python_version >= '3.0'",
"version": "==2.0.1"
},
"tqdm": {
"hashes": [
"sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
"sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
],
"version": "==4.51.0"
},
"twitter-scraper": {
"hashes": [
"sha256:03076748b01533f206f6b3e2e68c8a9b1f5ea167ff5fb4a4eb30e3000ad42f07",
"sha256:c58b6935166463f62cc3800a1c9a49212f4f948c97481d596a1d269ac349e88f"
],
"index": "pypi",
"version": "==0.4.4"
},
"urllib3": {
"hashes": [
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
],
"version": "==1.25.11"
},
"w3lib": {
"hashes": [
"sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53",
"sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"
],
"version": "==1.22.0"
},
"websockets": {
"hashes": [
"sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5",
"sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5",
"sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308",
"sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb",
"sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a",
"sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c",
"sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170",
"sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422",
"sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8",
"sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485",
"sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f",
"sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8",
"sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc",
"sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779",
"sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989",
"sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1",
"sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092",
"sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824",
"sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d",
"sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55",
"sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36",
"sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b"
],
"version": "==8.1"
}
},
"develop": {}
}

@ -0,0 +1,9 @@
#!/usr/bin/env python3
from twitter_scraper import get_tweets
def main():
for tweet in get_tweets('twitter', pages=1):
print(tweet['username'], ": ", tweet['text'])
if __name__ == "__main__":
main()

@ -0,0 +1,2 @@
run:
PYTHONUNBUFFERED=1 python3 twitter.py | tee -a users.csv

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -0,0 +1,61 @@
#!/usr/bin/env python3
import json
import sys
import requests
import time
import shelve
# those stupid pigfuckers at twitter don't include your followers or
# following lists in your twitter data export by username, only by numeric
# userid, so you can't easily take your social graph with you, if you e.g.
# want to use nitter to RSS follow certain twitter feeds.
# additionally, the export includes your twitter user lists: but only the
# names and IDs of the lists - *none of the actual userids or usernames that
# comprise the lists*. just that you have a list of id x and name y, not
# who's actually on it.
# twitter are fucking assholes.
# i deleted my account right after this export.
# https://sneak.berlin/20201031/goodbye-twitter/
EXPORT_DATE = "2020-11-08T01:07:55Z"
# thanks random php person
CONVERTER_URL = "https://tweeterid.com/ajax.php"
def main():
followers = json.load(open('followers.json'))
followings = json.load(open('followings.json'))
with shelve.open('usernamedb') as db:
def save_username(id,username):
if username != "error":
db[str(id)] = username
print("saved db[%s]='%s'" % (id, username))
else:
raise ValueError("api error");
for following in followings:
id = following['following']['accountId']
if db.get(str(id),None) is None:
time.sleep(1)
username = requests.post(CONVERTER_URL, { "input": id }).text.strip()
save_username(id,username)
else:
username = db.get(str(id))
print("already had db[%s]=%s" % (id, username))
for follower in followers:
id = follower['follower']['accountId']
if db.get(str(id),None) is None:
time.sleep(1)
username = requests.post(CONVERTER_URL, { "input": id }).text.strip()
save_username(id,username)
else:
username = db.get(str(id))
print("already had db[%s]=%s" % (id, username))
if __name__ == "__main__":
main()

Binary file not shown.

File diff suppressed because it is too large Load Diff

@ -0,0 +1,39 @@
#!/bin/bash
set -o nounset
set -o errexit
set -o pipefail
SD="/var/umbra"
function checkMounted() {
if [[ ! -e $SD/cur ]]; then
echo "umbraSync directory not mounted." > /dev/stderr
exit 2
fi
}
function writeStats() {
checkMounted
OD="$SD/host/$(hostname -s)/stat"
if [[ ! -d "$OD" ]]; then
mkdir -p "$OD"
fi
ifconfig > "$OD/ifconfig"
netstat -in > "$OD/ifaces"
date -u "+%s" > "$OD/cur"
}
function errorFileName () {
echo "$(date -u +"%Y-%m-%dT%H%M%SZ").$(hostname -s).$(uuidgen).error.txt"
}
function logError() {
checkMounted
OD="$SD/host/$(hostname -s)/error"
if [[ ! -d "$OD" ]]; then
mkdir -p "$OD"
fi
cat > "$OD/$(errorFileName)" <<< "$@"
}
Loading…
Cancel
Save