zhangyuqing
/
casnode
mirror of https://github.com/casbin/casnode.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve TestSyncThreads().

This commit is contained in:
Gucheng Wang 2021-11-21 13:55:45 +08:00
parent 8339480f32
commit 7058219a14
6 changed files with 132 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
discuzx/attachment.go
View File

@ -34,20 +34,10 @@ type Attachment struct {
Width int
}
//func getAttachments() []*Attachment {
// attachments := []*Attachment{}
// err := adapter.Engine.Table("forum_attachment").Find(&attachments)
// if err != nil {
// panic(err)
// }
//
// return attachments
//}
func getAttachmentsForThreadInTable(tableIndex int, threadId int) []*Attachment {
func getAttachmentsInTable(tableIndex int) []*Attachment {
attachments := []*Attachment{}
tableName := fmt.Sprintf("forum_attachment_%d", tableIndex)
err := adapter.Engine.Table(tableName).Where("tid = ?", threadId).Find(&attachments)
tableName := fmt.Sprintf("pre_forum_attachment_%d", tableIndex)
err := adapter.Engine.Table(tableName).Find(&attachments)
if err != nil {
panic(err)
}
@ -55,15 +45,29 @@ func getAttachmentsForThreadInTable(tableIndex int, threadId int) []*Attachment
return attachments
}
func getAttachmentsForThread(threadId int) []*Attachment {
func getAttachments() []*Attachment {
attachments := []*Attachment{}
for i := 0; i < 10; i++ {
tmp := getAttachmentsForThreadInTable(i, threadId)
tmp := getAttachmentsInTable(i)
attachments = append(attachments, tmp...)
}
return attachments
}
func getAttachmentMap() map[int][]*Attachment {
attachments := getAttachments()
m := map[int][]*Attachment{}
for _, attachment := range attachments {
if _, ok := m[attachment.Tid]; !ok {
m[attachment.Tid] = []*Attachment{}
}
m[attachment.Tid] = append(m[attachment.Tid], attachment)
}
return m
}
func uploadAttachmentAndUpdatePost(cdnDomain string, attachment *Attachment, postMap map[int]*Post) {
post := postMap[attachment.Pid]

56
discuzx/class.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright 2021 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package discuzx
type Class struct {
Typeid int
Fid int
Name string
Displayorder int
}
func getClasses() []*Class {
classes := []*Class{}
err := adapter.Engine.Table("pre_forum_threadclass").Find(&classes)
if err != nil {
panic(err)
}
return classes
}
func getClass(id int) *Class {
class := Class{Typeid: id}
existed, err := adapter.Engine.Table("pre_forum_threadclass").Get(&class)
if err != nil {
panic(err)
}
if existed {
return &class
} else {
return nil
}
}
func getClassMap() map[int]*Class {
classes := getClasses()
m := map[int]*Class{}
for _, class := range classes {
m[class.Typeid] = class
}
return m
}

10
discuzx/post.go
View File

@ -31,7 +31,7 @@ type Post struct {
func getPosts() []*Post {
posts := []*Post{}
err := adapter.Engine.Table("forum_post").Find(&posts)
err := adapter.Engine.Table("pre_forum_post").Find(&posts)
if err != nil {
panic(err)
}
@ -41,7 +41,7 @@ func getPosts() []*Post {
func getPostsForThread(threadId int) []*Post {
posts := []*Post{}
err := adapter.Engine.Table("forum_post").Where("tid = ?", threadId).Find(&posts)
err := adapter.Engine.Table("pre_forum_post").Where("tid = ?", threadId).Find(&posts)
if err != nil {
panic(err)
}
@ -56,3 +56,9 @@ func getPostMapFromPosts(posts []*Post) map[int]*Post {
}
return res
}
func getPostMapForThread(threadId int) ([]*Post, map[int]*Post) {
posts := getPostsForThread(threadId)
postMap := getPostMapFromPosts(posts)
return posts, postMap
}

23
discuzx/thread.go
View File

@ -15,16 +15,19 @@
package discuzx
type Thread struct {
Tid int
Fid int
Typeid int
Author string
Subject string
Dateline int
Lastpost int
Lastposter string
Views int
Replies int
Tid int
Fid int
Typeid int
Author string
Subject string
Dateline int
Lastpost int
Lastposter string
Views int
Replies int
RecommendAdd int
Heats int
Favtimes int
Posts []*Post `xorm:"-"`
}

46
discuzx/thread_test.go
View File

@ -43,56 +43,40 @@ func TestGetThreads(t *testing.T) {
thread := threadMap[126239]
println(thread)
addWholeTopic(thread)
//for _, thread := range threadMap {
// thread.
//}
addWholeTopic(thread, nil, nil)
}
func syncThread(threadId int) {
//threadId := 1
//threadId := 459
//threadId := 36643
//threadId := 126239
attachments := getAttachmentsForThread(threadId)
func syncThread(threadId int, attachments []*Attachment, forum *Forum, classMap map[int]*Class) {
thread := getThread(threadId)
if thread == nil {
return
}
posts := getPostsForThread(thread.Tid)
thread.Posts = append(thread.Posts, posts...)
postMap := getPostMapFromPosts(posts)
posts, postMap := getPostMapForThread(thread.Tid)
thread.Posts = posts
deleteWholeTopic(thread)
for _, attachment := range attachments {
uploadAttachmentAndUpdatePost(cdnDomain, attachment, postMap)
}
addWholeTopic(thread)
addWholeTopic(thread, forum, classMap)
}
func TestGetThread(t *testing.T) {
func TestSyncThreads(t *testing.T) {
object.InitConfig()
InitAdapter()
object.InitAdapter()
//syncThread(114)
attachmentMap := getAttachmentMap()
forumMap := getForumMap()
classMap := getClassMap()
threads := getThreads()
for _, thread := range threads {
if thread.Fid != 2 && thread.Fid != 100 && thread.Fid != 40 {
continue
}
if thread.Tid <= 125956 {
continue
}
syncThread(thread.Tid)
fmt.Printf("Processed thread, tid = %d, fid = %d\n", thread.Tid, thread.Fid)
for i, thread := range threads {
attachments := attachmentMap[thread.Tid]
forum := forumMap[thread.Fid]
syncThread(thread.Tid, attachments, forum, classMap)
fmt.Printf("[%d/%d]: Synced thread: tid = %d, fid = %d\n", i, len(threads), thread.Tid, thread.Fid)
}
//for i := 0; i < 10; i ++ {
// syncThread(i)
//}
}

66
discuzx/topic.go
View File

@ -16,47 +16,12 @@ package discuzx
import (
"fmt"
"strconv"
"github.com/casbin/casnode/object"
)
func getNodeFromThread(thread *Thread) *object.Node {
if thread.Fid == 2 {
if thread.Typeid == 23 {
return object.GetNode("resolved")
} else if thread.Typeid == 26 {
return object.GetNode("room")
} else if thread.Typeid == 27 {
return object.GetNode("qq-group")
} else if thread.Typeid == 28 {
return object.GetNode("forum-online")
} else if thread.Typeid == 30 {
return object.GetNode("academic")
} else if thread.Typeid == 35 {
return object.GetNode("other-category")
} else {
return object.GetNode("no-category")
}
} else if thread.Fid == 100 {
year := getYearFromUnixSeconds(thread.Dateline)
if year < 2019 {
year = 2019
}
return object.GetNode(fmt.Sprintf("read-%d", year))
} else if thread.Fid == 40 {
year := getYearFromUnixSeconds(thread.Dateline)
if year < 2019 {
year = 2019
}
return object.GetNode(fmt.Sprintf("show-%d", year))
} else {
return nil
}
}
func addTopic(thread *Thread) int {
node := getNodeFromThread(thread)
func addTopic(thread *Thread, forum *Forum, classMap map[int]*Class) int {
content := ""
if thread.Posts[0].First == 1 {
content = thread.Posts[0].Message
@ -66,22 +31,33 @@ func addTopic(thread *Thread) int {
content = escapeContent(content)
content = addAttachmentsToContent(content, thread.Posts[0].UploadFileRecords)
tags := []string{}
if class, ok := classMap[thread.Typeid]; ok {
tags = append(tags, class.Name)
}
nodeName := strconv.Itoa(thread.Fid)
if forum != nil {
nodeName = forum.Name
}
topic := object.Topic{
Author: thread.Author,
NodeId: node.Id,
NodeName: node.Name,
NodeId: nodeName,
NodeName: nodeName,
Title: thread.Subject,
CreatedTime: getTimeFromUnixSeconds(thread.Dateline),
Tags: nil,
Tags: tags,
LastReplyUser: thread.Lastposter,
LastReplyTime: getTimeFromUnixSeconds(thread.Lastpost),
ReplyCount: thread.Replies,
UpCount: 0,
UpCount: thread.RecommendAdd,
HitCount: thread.Views,
Hot: 0,
FavoriteCount: 0,
Hot: thread.Heats,
FavoriteCount: thread.Favtimes,
Deleted: false,
Content: content,
IsHidden: false,
}
res, id := object.AddTopic(&topic)
@ -165,7 +141,7 @@ func addAttachmentsToContent(content string, records []*object.UploadFileRecord)
return content
}
func addWholeTopic(thread *Thread) {
func addWholeTopic(thread *Thread, forum *Forum, classMap map[int]*Class) {
// remove leading useless posts
posts := []*Post{}
isBeforeFirstPosition := true
@ -182,7 +158,7 @@ func addWholeTopic(thread *Thread) {
return
}
topicId := addTopic(thread)
topicId := addTopic(thread, forum, classMap)
for i, post := range thread.Posts {
if i == 0 {
continue