Skip to content

Commit 80430ec

Browse files
asimasim
committed
Add usage server
1 parent f1c03b9 commit 80430ec

6 files changed

Lines changed: 579 additions & 0 deletions

File tree

‎cmd/usage/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Usage
2+
3+
Usage captures usage telemetry from micro
4+
5+
## Overview
6+
7+
Usage is a small net/http server backed by boltdb which captures micro usage telemetry
8+
9+
- Runs on port :8091
10+
- Receives requests to `/usage` in proto format
11+
- Backed by boltdb for stateful storage
12+
13+
## Usage
14+
15+
```
16+
go run main.go
17+
```

‎cmd/usage/client/main.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io/ioutil"
7+
"net/http"
8+
"os"
9+
"sort"
10+
"strings"
11+
)
12+
13+
type Result struct {
14+
Count map[string]int64 `json:"count"`
15+
}
16+
17+
func printKey(k string, v int64) {
18+
if v == 0 {
19+
return
20+
}
21+
22+
var u string
23+
var c float64
24+
25+
switch {
26+
case v > 1e9:
27+
c = float64(v) / 1e9
28+
u = "b"
29+
case v > 1e6:
30+
c = float64(v) / 1e6
31+
u = "m"
32+
case v > 1e4:
33+
c = float64(v) / 1e3
34+
u = "k"
35+
default:
36+
c = float64(v)
37+
}
38+
39+
fmt.Printf("micro %s:\t%.2f%s\n", k, c, u)
40+
}
41+
42+
func main() {
43+
var cKey string
44+
if len(os.Args) > 1 {
45+
cKey = os.Args[1]
46+
}
47+
48+
rsp, err := http.Get("https://micro.mu/usage?date=2019")
49+
if err != nil {
50+
fmt.Println(err)
51+
return
52+
}
53+
defer rsp.Body.Close()
54+
55+
b, err := ioutil.ReadAll(rsp.Body)
56+
if err != nil {
57+
fmt.Println(err)
58+
return
59+
}
60+
61+
var results map[string]Result
62+
63+
counts := map[string]int64{}
64+
highest := map[string]int64{}
65+
// daily := map[string]int64{}
66+
monthly := map[string]int64{}
67+
68+
if err := json.Unmarshal(b, &results); err != nil {
69+
fmt.Println(err)
70+
return
71+
}
72+
73+
for k, v := range results {
74+
// 20190520-micro.new
75+
parts := strings.Split(k, ".")
76+
if len(parts) < 2 {
77+
continue
78+
}
79+
80+
// micro.new
81+
key := parts[len(parts)-1]
82+
83+
if len(cKey) > 0 && key != cKey {
84+
continue
85+
}
86+
87+
// counts[micro.new] += requests
88+
c := counts[key]
89+
c += v.Count["requests"]
90+
// save
91+
counts[key] = c
92+
93+
// set highest
94+
if i := highest[key]; v.Count["requests"] > i {
95+
highest[key] = v.Count["requests"]
96+
}
97+
98+
// set monthly
99+
month := parts[0][:6]
100+
mkey := key + " (" + month + ")"
101+
c = monthly[mkey]
102+
c += v.Count["requests"]
103+
monthly[mkey] = c
104+
}
105+
106+
fmt.Println("Total requests:")
107+
108+
for k, v := range counts {
109+
printKey(k, v)
110+
}
111+
112+
fmt.Println("\nHighest requests:")
113+
114+
for k, v := range highest {
115+
printKey(k, v)
116+
}
117+
118+
fmt.Println("\nMonthly requests:")
119+
var keys []string
120+
for k, _ := range monthly {
121+
keys = append(keys, k)
122+
}
123+
124+
sort.Strings(keys)
125+
126+
for _, k := range keys {
127+
printKey(k, monthly[k])
128+
}
129+
}

‎cmd/usage/main.go‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"log"
9+
"net/http"
10+
"os"
11+
"strings"
12+
"sync"
13+
"time"
14+
15+
"github.com/boltdb/bolt"
16+
"github.com/golang/protobuf/proto"
17+
"github.com/gorilla/handlers"
18+
pb "github.com/micro/micro/cmd/usage/proto"
19+
)
20+
21+
var (
22+
db *bolt.DB
23+
fd = "usage.db"
24+
25+
mtx sync.RWMutex
26+
seen = map[string]uint64{}
27+
)
28+
29+
func setup() {
30+
// setup db
31+
d, err := bolt.Open(fd, 0600, &bolt.Options{Timeout: 1 * time.Second})
32+
if err != nil {
33+
log.Fatal(err)
34+
}
35+
db = d
36+
37+
if err := db.Update(func(tx *bolt.Tx) error {
38+
for _, b := range []string{"usage", "metrics"} {
39+
if _, err := tx.CreateBucketIfNotExists([]byte(b)); err != nil {
40+
return err
41+
}
42+
}
43+
return nil
44+
}); err != nil {
45+
log.Fatal(err)
46+
}
47+
48+
go flush()
49+
}
50+
51+
func flush() {
52+
for {
53+
time.Sleep(time.Hour)
54+
now := time.Now().UnixNano()
55+
mtx.Lock()
56+
for k, v := range seen {
57+
d := uint64(now) - v
58+
// 48 hours
59+
if d > 1.728e14 {
60+
delete(seen, k)
61+
}
62+
}
63+
seen = make(map[string]uint64)
64+
mtx.Unlock()
65+
}
66+
}
67+
68+
func process(w http.ResponseWriter, r *http.Request, u *pb.Usage) {
69+
today := time.Now().Format("20060102")
70+
key := fmt.Sprintf("%s-%s", u.Service, u.Id)
71+
now := uint64(time.Now().UnixNano())
72+
73+
mtx.Lock()
74+
last := seen[key]
75+
lastSeen := now - last
76+
seen[key] = now
77+
mtx.Unlock()
78+
79+
db.Update(func(tx *bolt.Tx) error {
80+
b := tx.Bucket([]byte(`usage`))
81+
buf, err := proto.Marshal(u)
82+
if err != nil {
83+
return err
84+
}
85+
k := fmt.Sprintf("%d-%s", u.Timestamp, key)
86+
// save this usage
87+
if err := b.Put([]byte(k), buf); err != nil {
88+
return err
89+
}
90+
91+
// save daily usage
92+
b = tx.Bucket([]byte(`metrics`))
93+
dailyKey := fmt.Sprintf("%s-%s", today, u.Service)
94+
95+
// get usage
96+
v := b.Get([]byte(dailyKey))
97+
if v == nil {
98+
// todo: don't overwrite this
99+
u.Metrics.Count["services"] = uint64(1)
100+
m, _ := proto.Marshal(u.Metrics)
101+
return b.Put([]byte(dailyKey), m)
102+
}
103+
104+
m := new(pb.Metrics)
105+
if err := proto.Unmarshal(v, m); err != nil {
106+
return err
107+
}
108+
109+
// update request count
110+
m.Count["requests"] += u.Metrics.Count["requests"]
111+
m.Count["services"] += u.Metrics.Count["services"]
112+
113+
// not seen today add it
114+
if lastSeen == 0 || lastSeen > 7.2e13 {
115+
c := m.Count["instances"]
116+
c++
117+
m.Count["instances"] = c
118+
}
119+
120+
buf, err = proto.Marshal(m)
121+
if err != nil {
122+
return err
123+
}
124+
125+
// store today-micro.api/new/cli/proxy
126+
return b.Put([]byte(dailyKey), buf)
127+
})
128+
}
129+
130+
func metrics(w http.ResponseWriter, r *http.Request) {
131+
r.ParseForm()
132+
prefix := time.Now().Add(time.Hour * -24).Format("20060102")
133+
metrics := map[string]interface{}{}
134+
135+
if date := r.Form.Get("date"); len(date) >= 4 && len(date) <= 8 {
136+
prefix = date
137+
}
138+
139+
db.View(func(tx *bolt.Tx) error {
140+
c := tx.Bucket([]byte(`metrics`)).Cursor()
141+
142+
for k, v := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() {
143+
m := new(pb.Metrics)
144+
proto.Unmarshal(v, m)
145+
key := strings.TrimPrefix(string(k), prefix+"-")
146+
metrics[key] = m
147+
}
148+
return nil
149+
})
150+
151+
var buf []byte
152+
ct := r.Header.Get("Content-Type")
153+
154+
if v := r.Form.Get("pretty"); len(v) > 0 || ct != "application/json" {
155+
buf, _ = json.MarshalIndent(metrics, "", "\t")
156+
} else {
157+
buf, _ = json.Marshal(metrics)
158+
}
159+
160+
if len(buf) == 0 {
161+
buf = []byte(`{}`)
162+
}
163+
164+
w.Header().Set("Content-Type", "application/json")
165+
w.Write(buf)
166+
}
167+
168+
func handler(w http.ResponseWriter, r *http.Request) {
169+
r.ParseForm()
170+
171+
// return metrics
172+
if r.Method == "GET" {
173+
metrics(w, r)
174+
return
175+
}
176+
177+
// require post for updates
178+
if r.Method != "POST" {
179+
return
180+
}
181+
if r.Header.Get("Content-Type") != "application/protobuf" {
182+
return
183+
}
184+
185+
if r.UserAgent() != "micro/usage" {
186+
return
187+
}
188+
189+
b, err := ioutil.ReadAll(r.Body)
190+
if err != nil {
191+
http.Error(w, err.Error(), 500)
192+
return
193+
}
194+
u := new(pb.Usage)
195+
if err := proto.Unmarshal(b, u); err != nil {
196+
http.Error(w, err.Error(), 500)
197+
return
198+
}
199+
go process(w, r, u)
200+
}
201+
202+
func main() {
203+
setup()
204+
http.HandleFunc("/", handler)
205+
206+
lh := handlers.LoggingHandler(os.Stdout, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
207+
if strings.HasPrefix(r.URL.Path, "/usage") {
208+
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/usage")
209+
}
210+
http.DefaultServeMux.ServeHTTP(w, r)
211+
}))
212+
213+
if err := http.ListenAndServe(":8091", lh); err != nil {
214+
log.Fatal(err)
215+
}
216+
}

‎cmd/usage/proto/usage.micro.go‎

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)