Global/wakapi
1
0
Fork 0
mirror of https://github.com/muety/wakapi.git synced 2025-12-06 06:22:41 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat: ability to exlclude unknown projects from summaries (resolve #619)

Browse Source
This commit is contained in:
Ferdinand Mütsch
2024-03-29 19:25:53 +01:00
parent 9d50b823ae
commit 34961e7c9a
13 changed files with 978 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1537
coverage/coverage.out
View File

File diff suppressed because it is too large Load Diff

5
mocks/user_service.go
View File

@@ -34,6 +34,11 @@ func (m *UserServiceMock) GetAll() ([]*models.User, error) {
return args.Get(0).([]*models.User), args.Error(1)
}
func (m *UserServiceMock) GetAllMapped() (map[string]*models.User, error) {
args := m.Called()
return args.Get(0).(map[string]*models.User), args.Error(1)
}
func (m *UserServiceMock) GetMany(s []string) ([]*models.User, error) {
args := m.Called(s)
return args.Get(0).([]*models.User), args.Error(1)

51
models/user.go
View File

@@ -15,31 +15,32 @@ func init() {
}
type User struct {
ID string `json:"id" gorm:"primary_key"`
ApiKey string `json:"api_key" gorm:"unique; default:NULL"`
Email string `json:"email" gorm:"index:idx_user_email; size:255"`
Location string `json:"location"`
Password string `json:"-"`
CreatedAt CustomTime `gorm:"default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
LastLoggedInAt CustomTime `gorm:"default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
ShareDataMaxDays int `json:"-"`
ShareEditors bool `json:"-" gorm:"default:false; type:bool"`
ShareLanguages bool `json:"-" gorm:"default:false; type:bool"`
ShareProjects bool `json:"-" gorm:"default:false; type:bool"`
ShareOSs bool `json:"-" gorm:"default:false; type:bool; column:share_oss"`
ShareMachines bool `json:"-" gorm:"default:false; type:bool"`
ShareLabels bool `json:"-" gorm:"default:false; type:bool"`
IsAdmin bool `json:"-" gorm:"default:false; type:bool"`
HasData bool `json:"-" gorm:"default:false; type:bool"`
WakatimeApiKey string `json:"-"` // for relay middleware and imports
WakatimeApiUrl string `json:"-"` // for relay middleware and imports
ResetToken string `json:"-"`
ReportsWeekly bool `json:"-" gorm:"default:false; type:bool"`
PublicLeaderboard bool `json:"-" gorm:"default:false; type:bool"`
SubscribedUntil *CustomTime `json:"-" gorm:"swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
SubscriptionRenewal *CustomTime `json:"-" gorm:"swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
StripeCustomerId string `json:"-"`
InvitedBy string `json:"-"`
ID string `json:"id" gorm:"primary_key"`
ApiKey string `json:"api_key" gorm:"unique; default:NULL"`
Email string `json:"email" gorm:"index:idx_user_email; size:255"`
Location string `json:"location"`
Password string `json:"-"`
CreatedAt CustomTime `gorm:"default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
LastLoggedInAt CustomTime `gorm:"default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
ShareDataMaxDays int `json:"-"`
ShareEditors bool `json:"-" gorm:"default:false; type:bool"`
ShareLanguages bool `json:"-" gorm:"default:false; type:bool"`
ShareProjects bool `json:"-" gorm:"default:false; type:bool"`
ShareOSs bool `json:"-" gorm:"default:false; type:bool; column:share_oss"`
ShareMachines bool `json:"-" gorm:"default:false; type:bool"`
ShareLabels bool `json:"-" gorm:"default:false; type:bool"`
IsAdmin bool `json:"-" gorm:"default:false; type:bool"`
HasData bool `json:"-" gorm:"default:false; type:bool"`
WakatimeApiKey string `json:"-"` // for relay middleware and imports
WakatimeApiUrl string `json:"-"` // for relay middleware and imports
ResetToken string `json:"-"`
ReportsWeekly bool `json:"-" gorm:"default:false; type:bool"`
PublicLeaderboard bool `json:"-" gorm:"default:false; type:bool"`
SubscribedUntil *CustomTime `json:"-" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
SubscriptionRenewal *CustomTime `json:"-" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"`
StripeCustomerId string `json:"-"`
InvitedBy string `json:"-"`
ExcludeUnknownProjects bool `json:"-"`
}
type Login struct {

45
repositories/user.go
View File

@@ -127,28 +127,29 @@ func (r *UserRepository) InsertOrGet(user *models.User) (*models.User, bool, err
func (r *UserRepository) Update(user *models.User) (*models.User, error) {
updateMap := map[string]interface{}{
"api_key": user.ApiKey,
"password": user.Password,
"email": user.Email,
"last_logged_in_at": user.LastLoggedInAt,
"share_data_max_days": user.ShareDataMaxDays,
"share_editors": user.ShareEditors,
"share_languages": user.ShareLanguages,
"share_oss": user.ShareOSs,
"share_projects": user.ShareProjects,
"share_machines": user.ShareMachines,
"share_labels": user.ShareLabels,
"wakatime_api_key": user.WakatimeApiKey,
"wakatime_api_url": user.WakatimeApiUrl,
"has_data": user.HasData,
"reset_token": user.ResetToken,
"location": user.Location,
"reports_weekly": user.ReportsWeekly,
"public_leaderboard": user.PublicLeaderboard,
"subscribed_until": user.SubscribedUntil,
"subscription_renewal": user.SubscriptionRenewal,
"stripe_customer_id": user.StripeCustomerId,
"invited_by": user.InvitedBy,
"api_key": user.ApiKey,
"password": user.Password,
"email": user.Email,
"last_logged_in_at": user.LastLoggedInAt,
"share_data_max_days": user.ShareDataMaxDays,
"share_editors": user.ShareEditors,
"share_languages": user.ShareLanguages,
"share_oss": user.ShareOSs,
"share_projects": user.ShareProjects,
"share_machines": user.ShareMachines,
"share_labels": user.ShareLabels,
"wakatime_api_key": user.WakatimeApiKey,
"wakatime_api_url": user.WakatimeApiUrl,
"has_data": user.HasData,
"reset_token": user.ResetToken,
"location": user.Location,
"reports_weekly": user.ReportsWeekly,
"public_leaderboard": user.PublicLeaderboard,
"subscribed_until": user.SubscribedUntil,
"subscription_renewal": user.SubscriptionRenewal,
"stripe_customer_id": user.StripeCustomerId,
"invited_by": user.InvitedBy,
"exclude_unknown_projects": user.ExcludeUnknownProjects,
}
result := r.db.Model(user).Updates(updateMap)

56
routes/settings.go
View File

@@ -39,6 +39,7 @@ type SettingsHandler struct {
keyValueSrvc services.IKeyValueService
mailSrvc services.IMailService
httpClient *http.Client
aggregationLocks map[string]bool
}
type action func(w http.ResponseWriter, r *http.Request) actionResult
@@ -77,6 +78,7 @@ func NewSettingsHandler(
keyValueSrvc: keyValueService,
mailSrvc: mailService,
httpClient: &http.Client{Timeout: 10 * time.Second},
aggregationLocks: make(map[string]bool),
}
}
@@ -178,6 +180,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
return h.actionDeleteUser
case "generate_invite":
return h.actionGenerateInvite
case "update_unknown_projects":
return h.actionUpdateExcludeUnknownProjects
}
return nil
}
@@ -298,6 +302,39 @@ func (h *SettingsHandler) actionUpdateLeaderboard(w http.ResponseWriter, r *http
return actionResult{http.StatusOK, "settings updated", "", nil}
}
func (h *SettingsHandler) actionUpdateExcludeUnknownProjects(w http.ResponseWriter, r *http.Request) actionResult {
if h.config.IsDev() {
loadTemplates()
}
var err error
user := middlewares.GetPrincipal(r)
defer h.userSrvc.FlushCache()
if h.isAggregationLocked(user.ID) {
return actionResult{http.StatusConflict, "", "summary regeneration already in progress, please wait", nil}
}
user.ExcludeUnknownProjects, err = strconv.ParseBool(r.PostFormValue("exclude_unknown_projects"))
if err != nil {
return actionResult{http.StatusBadRequest, "", "invalid input", nil}
}
if _, err := h.userSrvc.Update(user); err != nil {
return actionResult{http.StatusInternalServerError, "", "internal sever error", nil}
}
go func(user *models.User) {
h.toggleAggregationLock(user.ID, true)
defer h.toggleAggregationLock(user.ID, false)
if err := h.regenerateSummaries(user); err != nil {
conf.Log().Request(r).Error("failed to regenerate summaries for user '%s' - %v", user.ID, err)
}
}(user)
return actionResult{http.StatusOK, "regenerating summaries, this might take a while", "", nil}
}
func (h *SettingsHandler) actionUpdateSharing(w http.ResponseWriter, r *http.Request) actionResult {
if h.config.IsDev() {
loadTemplates()
@@ -627,11 +664,19 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
loadTemplates()
}
user := middlewares.GetPrincipal(r)
if h.isAggregationLocked(user.ID) {
return actionResult{http.StatusConflict, "", "summary regeneration already in progress, please wait", nil}
}
go func(user *models.User) {
h.toggleAggregationLock(user.ID, true)
defer h.toggleAggregationLock(user.ID, false)
if err := h.regenerateSummaries(user); err != nil {
conf.Log().Request(r).Error("failed to regenerate summaries for user '%s' - %v", user.ID, err)
}
}(middlewares.GetPrincipal(r))
}(user)
return actionResult{http.StatusAccepted, "summaries are being regenerated - this may take a up to a couple of minutes, please come back later", "", nil}
}
@@ -850,6 +895,15 @@ func (h *SettingsHandler) buildViewModel(r *http.Request, w http.ResponseWriter,
return routeutils.WithSessionMessages(vm, r, w)
}
func (h *SettingsHandler) toggleAggregationLock(userId string, locked bool) {
h.aggregationLocks[userId] = locked
}
func (h *SettingsHandler) isAggregationLocked(userId string) bool {
locked, _ := h.aggregationLocks[userId]
return locked
}
func getVal[T any](values *map[string]interface{}, key string, fallback T) T {
if values == nil {
return fallback

35
services/aggregation.go
View File

@@ -41,9 +41,9 @@ func NewAggregationService(userService IUserService, summaryService ISummaryServ
}
type AggregationJob struct {
UserID string
From time.Time
To time.Time
User *models.User
From time.Time
To time.Time
}
// Schedule a job to (re-)generate summaries every day shortly after midnight
@@ -96,25 +96,38 @@ func (srv *AggregationService) AggregateSummaries(userIds datastructure.Set[stri
if err := srv.queueWorkers.Dispatch(func() {
srv.process(job)
}); err != nil {
config.Log().Error("failed to dispatch summary generation job for user '%s'", job.UserID)
config.Log().Error("failed to dispatch summary generation job for user '%s'", job.User.ID)
}
}
}()
// Fetch complete user objects
var users map[string]*models.User
if userIds != nil && !userIds.IsEmpty() {
users, err = srv.userService.GetManyMapped(userIds.Values())
} else {
users, err = srv.userService.GetAllMapped()
}
if err != nil {
return err
}
// Generate summary aggregation jobs
for _, e := range lastUserSummaryTimes {
if userIds != nil && !userIds.IsEmpty() && !userIds.Contain(e.User) {
continue
}
u, _ := users[e.User]
if e.Time.Valid() {
// Case 1: User has aggregated summaries already
// -> Spawn jobs to create summaries from their latest aggregation to now
generateUserJobs(e.User, e.Time.T(), jobs)
generateUserJobs(u, e.Time.T(), jobs)
} else if t := firstUserHeartbeatLookup[e.User]; t.Valid() {
// Case 2: User has no aggregated summaries, yet, but has heartbeats
// -> Spawn jobs to create summaries from their first heartbeat to now
generateUserJobs(e.User, t.T(), jobs)
generateUserJobs(u, t.T(), jobs)
}
// Case 3: User doesn't have heartbeats at all
// -> Nothing to do
@@ -124,17 +137,17 @@ func (srv *AggregationService) AggregateSummaries(userIds datastructure.Set[stri
}
func (srv *AggregationService) process(job AggregationJob) {
if summary, err := srv.summaryService.Summarize(job.From, job.To, &models.User{ID: job.UserID}, nil); err != nil {
config.Log().Error("failed to generate summary (%v, %v, %s) - %v", job.From, job.To, job.UserID, err)
if summary, err := srv.summaryService.Summarize(job.From, job.To, job.User, nil); err != nil {
config.Log().Error("failed to generate summary (%v, %v, %s) - %v", job.From, job.To, job.User.ID, err)
} else {
logbuch.Info("successfully generated summary (%v, %v, %s)", job.From, job.To, job.UserID)
logbuch.Info("successfully generated summary (%v, %v, %s)", job.From, job.To, job.User.ID)
if err := srv.summaryService.Insert(summary); err != nil {
config.Log().Error("failed to save summary (%v, %v, %s) - %v", summary.UserID, summary.FromTime, summary.ToTime, err)
}
}
}
func generateUserJobs(userId string, from time.Time, jobs chan<- *AggregationJob) {
func generateUserJobs(user *models.User, from time.Time, jobs chan<- *AggregationJob) {
var to time.Time
// Go to next day of either user's first heartbeat or latest aggregation
@@ -157,7 +170,7 @@ func generateUserJobs(userId string, from time.Time, jobs chan<- *AggregationJob
0, 0, 0, 0,
from.Location(),
)
jobs <- &AggregationJob{userId, from, to}
jobs <- &AggregationJob{user, from, to}
from = to
}
}

4
services/duration.go
View File

@@ -93,6 +93,10 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
continue
}
if user.ExcludeUnknownProjects && d.Project == "" {
continue
}
// will only happen if two heartbeats with different hashes (e.g. different project) have the same timestamp
// that, in turn, will most likely only happen for mysql, where `time` column's precision was set to second for a while
// assume that two non-identical heartbeats with identical time are sub-second apart from each other, so round up to expectancy value

1
services/services.go
View File

@@ -135,6 +135,7 @@ type IUserService interface {
GetUserByResetToken(string) (*models.User, error)
GetUserByStripeCustomerId(string) (*models.User, error)
GetAll() ([]*models.User, error)
GetAllMapped() (map[string]*models.User, error)
GetMany([]string) ([]*models.User, error)
GetManyMapped([]string) (map[string]*models.User, error)
GetAllByReports(bool) ([]*models.User, error)

18
services/user.go
View File

@@ -119,6 +119,14 @@ func (srv *UserService) GetAll() ([]*models.User, error) {
return srv.repository.GetAll()
}
func (srv *UserService) GetAllMapped() (map[string]*models.User, error) {
users, err := srv.repository.GetAll()
if err != nil {
return nil, err
}
return srv.MapUsersById(users), nil
}
func (srv *UserService) GetMany(ids []string) ([]*models.User, error) {
return srv.repository.GetMany(ids)
}
@@ -128,9 +136,7 @@ func (srv *UserService) GetManyMapped(ids []string) (map[string]*models.User, er
if err != nil {
return nil, err
}
return convertor.ToMap[*models.User, string, *models.User](users, func(u *models.User) (string, *models.User) {
return u.ID, u
}), nil
return srv.MapUsersById(users), nil
}
func (srv *UserService) GetAllByReports(reportsEnabled bool) ([]*models.User, error) {
@@ -227,6 +233,12 @@ func (srv *UserService) Delete(user *models.User) error {
return srv.repository.Delete(user)
}
func (srv *UserService) MapUsersById(users []*models.User) map[string]*models.User {
return convertor.ToMap[*models.User, string, *models.User](users, func(u *models.User) (string, *models.User) {
return u.ID, u
})
}
func (srv *UserService) FlushCache() {
srv.cache.Flush()
}

6
static/assets/css/app.css
View File

@@ -189,4 +189,8 @@ body {
#summary-page svg rect {
cursor: help;
}
*/
*/
.wi-min {
width: min-content !important;
}

2
static/assets/css/app.dist.css
View File

File diff suppressed because one or more lines are too long

BIN
static/assets/css/app.dist.css.br
View File

Binary file not shown.

34
views/settings.tpl.html
View File

@@ -203,6 +203,39 @@
</div>
<div v-cloak id="data" class="tab flex flex-col space-y-4" v-if="isActive('data')">
<!-- Unknown Projects -->
<form class="w-full" action="" method="post">
<input type="hidden" name="action" value="update_unknown_projects">
<div class="flex flex-wrap md:flex-nowrap mb-2 gap-x-4">
<div class="w-full md:w-1/3 mb-2 md:mb-0 inline-block">
<span class="font-semibold text-gray-300 text-lg">Unknown Projects</span>
<p class="block text-sm text-gray-600">
You can choose to exclude and ignore coding stats from unknown projects. Changing this setting will require to recompute your statistics.
</p>
</div>
<div class="flex-col w-full md:w-2/3 inline-block space-y-4">
<div class="flex justify-between items-center">
<div class="flex flex-col gap-y-1">
<label class="font-semibold text-gray-300" for="share_projects">Exclude unknown projects</label>
<select autocomplete="off" id="unknown-projects-toggle" name="exclude_unknown_projects" class="select-default wi-min">
<option value="false" class="cursor-pointer" {{ if not .User.ExcludeUnknownProjects }} selected {{ end }}>No
</option>
<option value="true" class="cursor-pointer" {{ if .User.ExcludeUnknownProjects }} selected {{ end }}>Yes
</option>
</select>
</div>
<button type="submit" class="btn-primary h-min">Save</button>
</div>
</div>
</div>
</form>
<div class="w-full">
<hr class="border-t border-gray-800 my-4">
</div>
<!-- Aliases -->
<div class="w-full">
<div class="flex flex-wrap flex-nowrap mb-8 gap-x-4">
@@ -276,7 +309,6 @@
<hr class="border-t border-gray-800 my-4">
</div>
<!-- Project Labels -->
<div class="w-full">
<div class="flex flex-wrap md:flex-nowrap mb-8 gap-x-4">