Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Per-scraper tags, updated match/replace tag list #1

Closed
wants to merge 6 commits into from
Closed

Per-scraper tags, updated match/replace tag list #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b88e30f
Update javdatabase.go
vt-idiot Jan 9, 2023
b65e161
Update javbus.go
vt-idiot Jan 9, 2023
75232f1
Update javland.go
vt-idiot Jan 9, 2023
3ce2381
Update javlibrary.go
vt-idiot Jan 9, 2023
ac16d34
Update javtags.go
vt-idiot Jan 9, 2023
bc7ad2c
Update javtags.go
vt-idiot Jan 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions pkg/scrape/javbus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ func ScrapeJavBus(out *[]models.ScrapedScene, queryString string) {
// Always add 'javr' as a tag
sc.Tags = append(sc.Tags, `javr`)

// Always add 'javbus' as a tag
sc.Tags = append(sc.Tags, `javbus`)

html.ForEach(`div.row.movie div.info > p`, func(id int, p *colly.HTMLElement) {
label := p.ChildText(`span.header`)

Expand Down
3 changes: 3 additions & 0 deletions pkg/scrape/javdatabase.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ func ScrapeJavDB(out *[]models.ScrapedScene, queryString string) {

// Always add 'javr' as a tag
sc.Tags = append(sc.Tags, `javr`)

// Always add 'javdatabase' as a tag
sc.Tags = append(sc.Tags, `javdatabase`)

// Cast
html.ForEach("h2.subhead", func(id int, h2 *colly.HTMLElement) {
Expand Down
3 changes: 3 additions & 0 deletions pkg/scrape/javland.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ func ScrapeJavLand(out *[]models.ScrapedScene, queryString string) {

// Always add 'javr' as a tag
sc.Tags = append(sc.Tags, `javr`)

// Always add 'jav.land' as a tag
sc.Tags = append(sc.Tags, `jav.land`)

html.ForEach(`table.videotextlist tr`, func(id int, tr *colly.HTMLElement) {
tds := tr.DOM.Children()
Expand Down
3 changes: 3 additions & 0 deletions pkg/scrape/javlibrary.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ func ScrapeJavLibrary(out *[]models.ScrapedScene, queryString string) {
// Tags
// Always add 'javr' as a tag
sc.Tags = append(sc.Tags, `javr`)

// Always add 'javlibrary' as a tag
sc.Tags = append(sc.Tags, `javlibrary`)

// ID
videoIdSel := e.DOM.Find("div#video_id td.text")
Expand Down
35 changes: 18 additions & 17 deletions pkg/scrape/javtags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@ func ProcessJavrTag(tag string) string {

// Skipping some very generic and useless tags
skiptags := map[string]bool{
"featured actress": true,
"vr exclusive": true,
"high quality vr": true,
"high-quality vr": true,
"vr": true,
"vr only": true,
"hi-def": true,
"exclusive distribution": true,
"dmm exclusive": true,
"featured actress": true,
"vr exclusive": true,
"high quality vr": true,
"high-quality vr": true,
"vr": true,
"vr only": true,
"hi-def": true,
"exclusive distribution": true,
"single work": true,
"solo work": true,
"solowork": true,
}
if skiptags[taglower] {
return ""
Expand All @@ -30,14 +32,13 @@ func ProcessJavrTag(tag string) string {
// Map some tags to normalize so different sources match
// TODO: this mapping is totally incomplete and needs help from community to fill
maptags := map[string]string{
"blow": "blowjob",
"blow job": "blowjob",
"kiss kiss": "kiss",
"kiss / kiss": "kiss",
"prostitute": "club hostess & sex worker",
"prostitutes": "club hostess & sex worker",
"single work": "solo work",
"suntan": "sun tan",
"blow": "blowjob",
"blow job": "blowjob",
"kiss": "kiss kiss",
"kiss / kiss": "kiss kiss",
"prostitute": "club hostess & sex worker",
"prostitutes": "club hostess & sex worker",
"sun tan": "suntan",
}
if maptags[taglower] != "" {
return maptags[taglower]
Expand Down