databaseRead.go 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. package lib
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. )
  6. //topicPointer and textPointer should not be exported
  7. //mb move tweet reconstruction to dbRead from servers
  8. type Tweet struct {
  9. TopicPointer string
  10. TextPointer int
  11. Topics []string
  12. Text string
  13. RoundPosted int
  14. }
  15. var dbR = make(map[string][]Tweet)
  16. var archive = make(map[string][]Tweet)
  17. //has to be dividable by 32
  18. var minimumBlockSize int
  19. const roundsBeforeArchiving = 3
  20. var topicList []string
  21. var archiveTopicList []string
  22. func NewEntries(inputTweets []Tweet, whereTo int) {
  23. tmpdb := dbR
  24. if whereTo == 1 {
  25. tmpdb = archive
  26. }
  27. var position int = 0
  28. for _, tweet := range inputTweets {
  29. for index := range tweet.Topics {
  30. //new topic
  31. if _, ok := tmpdb[tweet.Topics[index]]; !ok {
  32. if whereTo == 0 {
  33. topicList = append(topicList, tweet.Topics[index])
  34. } else {
  35. archiveTopicList = append(archiveTopicList, tweet.Topics[index])
  36. }
  37. }
  38. //new tweet
  39. if index == 0 {
  40. position = len(tmpdb[tweet.Topics[0]])
  41. tmpdb[tweet.Topics[index]] = append(tmpdb[tweet.Topics[0]], tweet)
  42. } else {
  43. //known tweet
  44. //setting pointer for all other Topics
  45. topic := tweet.Topics[index]
  46. tweet.TopicPointer = tweet.Topics[0]
  47. tweet.TextPointer = position
  48. tweet.Topics = nil
  49. tweet.Text = ""
  50. tmpdb[topic] = append(tmpdb[topic], tweet)
  51. }
  52. }
  53. }
  54. if whereTo == 0 {
  55. dbR = tmpdb
  56. } else {
  57. archive = tmpdb
  58. }
  59. }
  60. //todo! add round to pirquery only get tweets that have been posted from that round onward
  61. func GetTweets(pirQuery []byte, dataLength int, whereFrom int) []byte {
  62. tmpdb := dbR
  63. if whereFrom == 1 {
  64. tmpdb = archive
  65. }
  66. minimumBlockSize = dataLength * maxTweetAmount(whereFrom)
  67. var wantedTopics = getNamesForTopics(pirQuery, whereFrom)
  68. tweetsToReturn := make([][]Tweet, len(wantedTopics))
  69. for index, wantedTopic := range wantedTopics {
  70. for _, tweet := range tmpdb[wantedTopic] {
  71. //new Tweet
  72. if tweet.Text != "" {
  73. tweet.RoundPosted = 0
  74. tweetsToReturn[index] = append(tweetsToReturn[index], tweet)
  75. } else {
  76. //"copied" tweet
  77. //find tweet with pointers
  78. tweet = tmpdb[tweet.TopicPointer][tweet.TextPointer]
  79. tweet.RoundPosted = 0
  80. tweetsToReturn[index] = append(tweetsToReturn[index], tweet)
  81. }
  82. }
  83. }
  84. return tweetsToByteArray(tweetsToReturn)
  85. }
  86. func maxTweetAmount(whereFrom int) int {
  87. tmpdb := dbR
  88. if whereFrom == 1 {
  89. tmpdb = archive
  90. }
  91. var max int = 0
  92. for i := range tmpdb {
  93. nrOfTweets := len(dbR[i])
  94. if nrOfTweets > max {
  95. max = nrOfTweets
  96. }
  97. }
  98. return max
  99. }
  100. func getNamesForTopics(wantedIndices []byte, whereFrom int) []string {
  101. var topicNames []string
  102. tmpTopicList := topicList
  103. if whereFrom == 1 {
  104. tmpTopicList = archiveTopicList
  105. }
  106. for index, element := range wantedIndices {
  107. if element == 1 {
  108. topicNames = append(topicNames, tmpTopicList[index])
  109. }
  110. }
  111. return topicNames
  112. }
  113. //transform struct to byte array for sending
  114. func tweetsToByteArray(tweetsToReturn [][]Tweet) []byte {
  115. tweetsAsBytes := make([]byte, minimumBlockSize)
  116. for _, block := range tweetsToReturn {
  117. var tweetToAppend []byte
  118. for _, tweet := range block {
  119. for _, topic := range tweet.Topics {
  120. tweetToAppend = append(tweetToAppend, []byte(topic)...)
  121. tweetToAppend = append(tweetToAppend, ","...)
  122. }
  123. //replaces last with ";", bc there is text following and not another topic
  124. tweetToAppend = tweetToAppend[:len(tweetToAppend)-1]
  125. tweetToAppend = append(tweetToAppend, []byte(";")[0])
  126. tweetToAppend = append(tweetToAppend, []byte(tweet.Text)...)
  127. tweetToAppend = append(tweetToAppend, []byte(";")[0])
  128. }
  129. //adds padding
  130. for i := len(tweetToAppend); i < minimumBlockSize; i++ {
  131. //todo! replace with random chars
  132. tweetToAppend = append(tweetToAppend, []byte(";")[0])
  133. }
  134. Xor(tweetToAppend, tweetsAsBytes)
  135. }
  136. return tweetsAsBytes
  137. }
  138. //see func name
  139. func GetTopicList(whereFrom int) ([]byte, int) {
  140. tmpTopicList := topicList
  141. if whereFrom == 1 {
  142. tmpTopicList = archiveTopicList
  143. }
  144. if (len(tmpTopicList)) == 0 {
  145. return nil, 0
  146. }
  147. topicByteArray := new(bytes.Buffer)
  148. json.NewEncoder(topicByteArray).Encode(tmpTopicList)
  149. return topicByteArray.Bytes(), len(tmpTopicList)
  150. }
  151. //iterates through full dbR and moves old tweets to archive
  152. func CleanUpdbR(round int) {
  153. var tweetsToArchive []Tweet
  154. for j := range dbR {
  155. tweets := dbR[j]
  156. for i := len(tweets) - 1; i >= 0; i-- {
  157. if round-roundsBeforeArchiving == tweets[i].RoundPosted {
  158. //only adds the tweet to the archive when there is text
  159. if tweets[i].Text != "" {
  160. tweetsToArchive = append(tweetsToArchive, tweets[i])
  161. }
  162. //delets the tweet from the array
  163. tweets = append(tweets[:i], tweets[i+1:]...)
  164. }
  165. }
  166. dbR[j] = tweets
  167. }
  168. NewEntries(tweetsToArchive, 1)
  169. }