package lib import ( "bytes" "encoding/json" "sort" ) //topicPointer and textPointer should not be exported //mb move tweet reconstruction to dbRead from servers type Tweet struct { TopicPointer string TextPointer int Topics []string Text string RoundPosted int } var dbR = make(map[string][]Tweet) var archive = make(map[string][]Tweet) //needs to be dividable by roundsBeforUpdate var roundsBeforeArchiving = 2 var roundsBeforeArchivingInc = roundsBeforeArchiving var bytesSaved float64 var topicList []string var archiveTopicList []string func NewEntries(inputTweets []Tweet, whereTo int) { tmpdb := dbR if whereTo == 1 { tmpdb = archive } var position int = 0 for _, tweet := range inputTweets { for index := range tweet.Topics { //new topic if _, ok := tmpdb[tweet.Topics[index]]; !ok { if whereTo == 0 { topicList = append(topicList, tweet.Topics[index]) } else { archiveTopicList = append(archiveTopicList, tweet.Topics[index]) } } //new tweet if index == 0 { position = len(tmpdb[tweet.Topics[0]]) tmpdb[tweet.Topics[index]] = append(tmpdb[tweet.Topics[0]], tweet) } else { //known tweet //setting pointer for all other Topics if whereTo == 1 { bytesSaved += float64(len(tweet.Topics)) + float64(len(tweet.Text)) } topic := tweet.Topics[index] var pointerTweet Tweet pointerTweet.TopicPointer = tweet.Topics[0] pointerTweet.TextPointer = position pointerTweet.Topics = nil pointerTweet.Text = "" tmpdb[topic] = append(tmpdb[topic], pointerTweet) } } } if whereTo == 0 { dbR = tmpdb } else { archive = tmpdb } } func GetTweets(pirQuery []byte, dataLength int, whereFrom int, pubKey [32]byte) []byte { tmpdb := dbR if whereFrom == 1 { tmpdb = archive } var wantedTopics = getNamesForTopics(pirQuery, whereFrom) tweetsToReturn := make([][]Tweet, len(wantedTopics)) for index, wantedTopic := range wantedTopics { for _, tweet := range tmpdb[wantedTopic] { //new Tweet if tweet.Text != "" { tweet.RoundPosted = 0 tweetsToReturn[index] = append(tweetsToReturn[index], tweet) } else { //"copied" tweet //find tweet with pointers tweet = tmpdb[tweet.TopicPointer][tweet.TextPointer] tweet.RoundPosted = 0 tweetsToReturn[index] = append(tweetsToReturn[index], tweet) } } } return tweetsToByteArray(tweetsToReturn, whereFrom, wantedTopics, dataLength) } func getNamesForTopics(wantedIndices []byte, whereFrom int) []string { var topicNames []string tmpTopicList := topicList if whereFrom == 1 { tmpTopicList = archiveTopicList } for index, element := range wantedIndices { if index == len(tmpTopicList) { break } if element == 1 { topicNames = append(topicNames, tmpTopicList[index]) } } return topicNames } //transform struct to byte array for sending func tweetsToByteArray(tweetsToReturn [][]Tweet, whereFrom int, wantedTopics []string, dataLength int) []byte { minimumBlockSize := dataLength * maxLength(whereFrom) tweetsAsBytes := make([]byte, minimumBlockSize) for _, block := range tweetsToReturn { var topicPadding []string var blockToAppend []byte for index, tweet := range block { for topicIndex, topic := range tweet.Topics { blockToAppend = append(blockToAppend, []byte(topic)...) blockToAppend = append(blockToAppend, ","...) //gets the topic used for padding if topicIndex > 0 && index < len(wantedTopics) && topic != wantedTopics[index] { topicPadding = append(topicPadding, topic) } } //replaces last "," with ";" bc there is text following and not another topic blockToAppend = blockToAppend[:len(blockToAppend)-1] blockToAppend = append(blockToAppend, []byte(";")[:]...) blockToAppend = append(blockToAppend, []byte(tweet.Text)...) blockToAppend = append(blockToAppend, []byte(";;")[:]...) } //adds padding blockToAppend = append(blockToAppend, []byte(";;;")[:]...) remainingLength := minimumBlockSize - len(blockToAppend) //grouping using topic from recovered tweets index := len(topicPadding) - 1 for index >= 0 && remainingLength > 0 && topicPadding[index] != "" { paddingTweet, err := getNextTweet(topicPadding[index], index, whereFrom) if err { break } if remainingLength < len(paddingTweet) { break } blockToAppend = append(blockToAppend, paddingTweet...) remainingLength -= len(paddingTweet) index-- } padding := bytes.Repeat([]byte(";"), remainingLength) blockToAppend = append(blockToAppend, padding...) Xor(blockToAppend, tweetsAsBytes) } return tweetsAsBytes } func maxLength(whereFrom int) int { tmpdb := dbR if whereFrom == 1 { tmpdb = archive } var max int = 0 for i := range tmpdb { nrOfTweets := len(tmpdb[i]) if nrOfTweets > max { max = nrOfTweets } } return max } //gets the Tweet at index from wantedTopic for padding func getNextTweet(wantedTopic string, index, whereFrom int) ([]byte, bool) { tmpdb := dbR if whereFrom == 1 { tmpdb = archive } var tweetToReturn Tweet for tweetIndex, tweet := range tmpdb[wantedTopic] { if len(tmpdb[wantedTopic]) <= index { return nil, true } else if tweetIndex < index { continue } else if tweetIndex > index { break } //new Tweet if tweet.Text != "" { tweet.RoundPosted = 0 tweetToReturn = tweet } else { //"copied" tweet //find tweet with pointers tweet = tmpdb[tweet.TopicPointer][tweet.TextPointer] tweet.RoundPosted = 0 tweetToReturn = tweet } } var tweetToReturnBytes []byte for _, topic := range tweetToReturn.Topics { tweetToReturnBytes = append(tweetToReturnBytes, []byte(topic)...) tweetToReturnBytes = append(tweetToReturnBytes, ","...) } //replaces last "," with ";;" bc there is text following and not another topic tweetToReturnBytes = tweetToReturnBytes[:len(tweetToReturnBytes)-1] tweetToReturnBytes = append(tweetToReturnBytes, []byte(";;")[:]...) tweetToReturnBytes = append(tweetToReturnBytes, []byte(tweetToReturn.Text)...) tweetToReturnBytes = append(tweetToReturnBytes, []byte(";")[0]) return tweetToReturnBytes, false } //see func name func GetTopicList(whereFrom int) ([]byte, int) { tmpTopicList := topicList if whereFrom == 1 { tmpTopicList = archiveTopicList } if (len(tmpTopicList)) == 0 { return nil, 0 } topicByteArray := new(bytes.Buffer) json.NewEncoder(topicByteArray).Encode(tmpTopicList) return topicByteArray.Bytes(), len(tmpTopicList) } //iterates through full dbR and moves old tweets to archive func CleanUpdbR(round int) { if roundsBeforeArchiving == -1 { return } if roundsBeforeArchivingInc-round == 0 { keys := make([]string, len(dbR)) i := 0 for k := range dbR { keys[i] = k i++ } sort.Strings(keys) var tweetsToArchive []Tweet for _, topic := range keys { tweets := dbR[topic] for i := len(tweets) - 1; i >= 0; i-- { if round-roundsBeforeArchiving >= tweets[i].RoundPosted { //only adds the tweet to the archive when there is text if tweets[i].Text != "" { tweetsToArchive = append(tweetsToArchive, tweets[i]) } //delets the tweet from the array row := append(tweets[:i], tweets[i+1:]...) tweets = row } } dbR[topic] = tweets } NewEntries(tweetsToArchive, 1) roundsBeforeArchivingInc += roundsBeforeArchiving //redoes the whole dbR to correct pointers var tweetsToMain []Tweet for _, topic := range keys { tweets := dbR[topic] for _, tweet := range tweets { if tweet.Text != "" { tweetsToMain = append(tweetsToMain, tweet) } } } dbR = nil dbR = make(map[string][]Tweet) topicList = nil NewEntries(tweetsToMain, 0) } } func GetBytesSaved() float64 { return bytesSaved / float64(len(archive)) }