Skip to content

Commit

Permalink
support new account discriminators
Browse files Browse the repository at this point in the history
  • Loading branch information
@ccbrown
ccbrown committed Nov 20, 2024
1 parent 2f43da4 commit 293352b
Show file tree
Hide file tree
Showing 5 changed files with 126 additions and 51 deletions.
112 changes: 91 additions & 21 deletions server/forum_indexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,23 +41,92 @@ func (indexer *ForumIndexer) Close() {
close(indexer.closeSignal)
}

type ForumAccount struct {
Username string
Discriminator int
}

func (indexer *ForumIndexer) run() {
log.Info("starting forum indexer")

accounts := []string{
"Chris", "Jonathan", "Erik", "Mark_GGG", "Samantha", "Rory", "Rhys", "Andrew_GGG",
"Damien_GGG", "Joel_GGG", "Ari", "Thomas", "BrianWeissman", "Edwin_GGG", "Support", "Dylan",
"MaxS", "Ammon_GGG", "Jess_GGG", "Robbie_GGG", "GGG_Neon", "Jason_GGG", "Henry_GGG",
"Michael_GGG", "Bex_GGG", "Cagan_GGG", "Kieren_GGG", "Yeran_GGG", "Gary_GGG", "Dan_GGG",
"Jared_GGG", "Brian_GGG", "RobbieL_GGG", "Arthur_GGG", "NickK_GGG", "Felipe_GGG",
"Alex_GGG", "Alexcc_GGG", "Andy", "CJ_GGG", "Eben_GGG", "Emma_GGG", "Ethan_GGG",
"Fitzy_GGG", "Hartlin_GGG", "Jake_GGG", "Melissa_GGG", "MikeP_GGG", "Novynn", "Rob_GGG",
"Roman_GGG", "Tom_GGG", "Natalia_GGG", "Jeff_GGG", "Lu_GGG", "JuliaS_GGG", "Alexander_GGG",
"SamC_GGG", "AndrewE_GGG", "Kyle_GGG", "Stacey_GGG", "Jatin_GGG", "Community_Team",
"Nick_GGG", "Guy_GGG", "Ben_GGG", "BenH_GGG", "Nav_GGG", "Will_GGG", "Scott_GGG", "JC_GGG",
"Dylan_GGG", "Chulainn_GGG", "Vash_GGG", "Cameron_GGG", "Jacob_GGG", "Jenn_GGG",
"CoryA_GGG", "Sian_GGG", "Drew_GGG", "Lisa_GGG", "ThomasK_GGG", "Whai_GGG", "Scopey",
"Adam_GGG", "Nichelle_GGG", "Markus_GGG",
accounts := []ForumAccount{
{Username: "Chris"},
{Username: "Jonathan"},
{Username: "Mark_GGG"},
{Username: "Rory"},
{Username: "Rhys"},
{Username: "Joel_GGG"},
{Username: "Ari"},
{Username: "Thomas"},
{Username: "Support"},
{Username: "Jess_GGG"},
{Username: "Robbie_GGG"},
{Username: "GGG_Neon"},
{Username: "Jason_GGG"},
{Username: "Henry_GGG"},
{Username: "Michael_GGG"},
{Username: "Bex_GGG"},
{Username: "Cagan_GGG"},
{Username: "Kieren_GGG"},
{Username: "Yeran_GGG"},
{Username: "Gary_GGG"},
{Username: "Dan_GGG"},
{Username: "Jared_GGG"},
{Username: "Brian_GGG"},
{Username: "RobbieL_GGG"},
{Username: "Arthur_GGG"},
{Username: "NickK_GGG"},
{Username: "Felipe_GGG"},
{Username: "Alex_GGG"},
{Username: "Alexcc_GGG"},
{Username: "CJ_GGG"},
{Username: "Eben_GGG"},
{Username: "Emma_GGG"},
{Username: "Ethan_GGG"},
{Username: "Fitzy_GGG"},
{Username: "Hartlin_GGG"},
{Username: "Jake_GGG"},
{Username: "Melissa_GGG"},
{Username: "MikeP_GGG"},
{Username: "Novynn"},
{Username: "Rob_GGG"},
{Username: "Roman_GGG"},
{Username: "Tom_GGG"},
{Username: "Natalia_GGG"},
{Username: "Jeff_GGG"},
{Username: "Lu_GGG"},
{Username: "JuliaS_GGG"},
{Username: "Alexander_GGG"},
{Username: "SamC_GGG"},
{Username: "AndrewE_GGG"},
{Username: "Kyle_GGG"},
{Username: "Stacey_GGG"},
{Username: "Jatin_GGG"},
{Username: "Community_Team"},
{Username: "Nick_GGG"},
{Username: "Guy_GGG"},
{Username: "Ben_GGG"},
{Username: "BenH_GGG"},
{Username: "Nav_GGG"},
{Username: "Will_GGG"},
{Username: "Scott_GGG"},
{Username: "JC_GGG"},
{Username: "Dylan_GGG"},
{Username: "Chulainn_GGG"},
{Username: "Vash_GGG"},
{Username: "Cameron_GGG"},
{Username: "Jacob_GGG"},
{Username: "Jenn_GGG"},
{Username: "CoryA_GGG"},
{Username: "Sian_GGG"},
{Username: "Drew_GGG"},
{Username: "Lisa_GGG"},
{Username: "ThomasK_GGG"},
{Username: "Whai_GGG"},
{Username: "Scopey"},
{Username: "Adam_GGG"},
{Username: "Nichelle_GGG"},
{Username: "Markus_GGG"},
}

timezone := (*time.Location)(nil)
Expand Down Expand Up @@ -99,7 +168,7 @@ func (indexer *ForumIndexer) run() {
return
default:
if err := indexer.index(account, timezone); err != nil {
log.WithError(err).Error("error indexing forum account: " + account)
log.WithError(err).Error("error indexing forum account: " + account.Username)
}
time.Sleep(time.Second)
}
Expand Down Expand Up @@ -143,7 +212,7 @@ var postURLExpression = regexp.MustCompile("^/forum/view-post/([0-9]+)")
var threadURLExpression = regexp.MustCompile("^/forum/view-thread/([0-9]+)")
var forumURLExpression = regexp.MustCompile("^/forum/view-forum/([0-9]+)")

func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumPost, error) {
func ScrapeForumPosts(doc *goquery.Document, poster ForumAccount, timezone *time.Location) ([]*ForumPost, error) {
posts := []*ForumPost(nil)

err := error(nil)
Expand All @@ -154,7 +223,8 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP

doc.Find(".forumPostListTable > tbody > tr").EachWithBreak(func(i int, sel *goquery.Selection) bool {
post := &ForumPost{
Poster: sel.Find(".post_by_account").Text(),
Poster: poster.Username,
PosterDiscriminator: poster.Discriminator,
}

body, err := sel.Find(".content").Html()
Expand Down Expand Up @@ -197,19 +267,19 @@ func ScrapeForumPosts(doc *goquery.Document, timezone *time.Location) ([]*ForumP
return posts, nil
}

func (indexer *ForumIndexer) forumPosts(poster string, page int, timezone *time.Location) ([]*ForumPost, error) {
doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v/page/%v", poster, page))
func (indexer *ForumIndexer) forumPosts(poster ForumAccount, page int, timezone *time.Location) ([]*ForumPost, error) {
doc, err := indexer.requestDocument(fmt.Sprintf("/account/view-posts/%v-%04d/page/%v", poster.Username, poster.Discriminator, page))
if err != nil {
return nil, err
}
posts, err := ScrapeForumPosts(doc, timezone)
posts, err := ScrapeForumPosts(doc, poster, timezone)
if err != nil {
return nil, err
}
return posts, nil
}

func (indexer *ForumIndexer) index(poster string, timezone *time.Location) error {
func (indexer *ForumIndexer) index(poster ForumAccount, timezone *time.Location) error {
logger := log.WithFields(log.Fields{
"poster": poster,
})
Expand Down
6 changes: 4 additions & 2 deletions server/forum_indexer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ func TestScrapeForumPosts(t *testing.T) {
tz, err := time.LoadLocation("America/Los_Angeles")
require.NoError(t, err)

posts, err := ScrapeForumPosts(doc, tz)
poster := ForumAccount{Username: "Chris"}
posts, err := ScrapeForumPosts(doc, poster, tz)
require.NoError(t, err)
assert.Equal(t, 10, len(posts))

Expand All @@ -31,6 +32,7 @@ func TestScrapeForumPosts(t *testing.T) {
assert.Equal(t, 54, p.ForumId)
assert.Equal(t, 1830139, p.ThreadId)
assert.Equal(t, "Chris", p.Poster)
assert.Equal(t, 0, p.PosterDiscriminator)
assert.Equal(t, "Photos of the Fan Meetup", p.ThreadTitle)
assert.Equal(t, "Announcements", p.ForumName)
assert.Equal(t, "we had a great ti<strong>m</strong>e too!", p.BodyHTML)
Expand All @@ -47,7 +49,7 @@ func TestScrapeForumPosts(t *testing.T) {
tz, err := time.LoadLocation("America/Los_Angeles")
require.NoError(t, err)

posts, err := ScrapeForumPosts(doc, tz)
posts, err := ScrapeForumPosts(doc, poster, tz)
require.Error(t, err)
assert.Equal(t, 0, len(posts))
})
Expand Down
55 changes: 29 additions & 26 deletions server/forum_post.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,39 +7,42 @@ import (
)

type ForumPost struct {
Id int `json:"id"`
BodyHTML string `json:"body_html"`
Time time.Time `json:"time"`
Poster string `json:"poster"`
ThreadId int `json:"thread_id"`
ThreadTitle string `json:"thread_title"`
ForumId int `json:"forum_id"`
ForumName string `json:"forum_name"`
Id int `json:"id"`
BodyHTML string `json:"body_html"`
Time time.Time `json:"time"`
Poster string `json:"poster"`
PosterDiscriminator int `json:"poster_discriminator"`
ThreadId int `json:"thread_id"`
ThreadTitle string `json:"thread_title"`
ForumId int `json:"forum_id"`
ForumName string `json:"forum_name"`
}

type forumPostWithHost struct {
Id int `json:"id"`
BodyHTML string `json:"body_html"`
Time time.Time `json:"time"`
Poster string `json:"poster"`
ThreadId int `json:"thread_id"`
ThreadTitle string `json:"thread_title"`
ForumId int `json:"forum_id"`
ForumName string `json:"forum_name"`
Host string `json:"host"`
Id int `json:"id"`
BodyHTML string `json:"body_html"`
Time time.Time `json:"time"`
Poster string `json:"poster"`
PosterDiscriminator int `json:"poster_discriminator"`
ThreadId int `json:"thread_id"`
ThreadTitle string `json:"thread_title"`
ForumId int `json:"forum_id"`
ForumName string `json:"forum_name"`
Host string `json:"host"`
}

func (p ForumPost) MarshalJSON() ([]byte, error) {
return json.Marshal(&forumPostWithHost{
Id: p.Id,
BodyHTML: p.BodyHTML,
Time: p.Time,
Poster: p.Poster,
ThreadId: p.ThreadId,
ThreadTitle: p.ThreadTitle,
ForumId: p.ForumId,
ForumName: p.ForumName,
Host: p.Host(),
Id: p.Id,
BodyHTML: p.BodyHTML,
Time: p.Time,
Poster: p.Poster,
PosterDiscriminator: p.PosterDiscriminator,
ThreadId: p.ThreadId,
ThreadTitle: p.ThreadTitle,
ForumId: p.ForumId,
ForumName: p.ForumName,
Host: p.Host(),
})
}

Expand Down
2 changes: 1 addition & 1 deletion server/index_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ var index = `<!DOCTYPE html><html>
</footer>
</div>
<script src="static/index.js?v7"></script>
<script src="static/index.js?v8"></script>
</body>
</html>`

Expand Down
2 changes: 1 addition & 1 deletion server/static/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ function loadActivity() {

if (type == 'forum_post') {
$tr.append($('<td class="poster">').append($('<a>')
.attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster))
.attr('href', 'https://' + activity.host + '/account/view-profile/' + encodeURIComponent(activity.poster) + '-' + String(activity.poster_discriminator || 0).padStart(4, '0'))
.text(activity.poster)
));
} else {
Expand Down

0 comments on commit 293352b

Please sign in to comment.