diff --git a/pkg/modules/auth/ldap/ldap.go b/pkg/modules/auth/ldap/ldap.go
index 1783de0e2..39766df90 100644
--- a/pkg/modules/auth/ldap/ldap.go
+++ b/pkg/modules/auth/ldap/ldap.go
@@ -28,6 +28,7 @@ import (
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/modules/auth"
+	"code.vikunja.io/api/pkg/modules/avatar"
 	"code.vikunja.io/api/pkg/modules/avatar/upload"
 	"code.vikunja.io/api/pkg/user"
 	"code.vikunja.io/api/pkg/utils"
@@ -187,6 +188,7 @@ func AuthenticateUserInLDAP(s *xorm.Session, username, password string, syncGrou
 			if err != nil {
 				return nil, err
 			}
+			avatar.FlushAllCaches(u)
 		}
 	}
 
diff --git a/pkg/modules/auth/openid/openid.go b/pkg/modules/auth/openid/openid.go
index 71a7804a6..b2eb982b5 100644
--- a/pkg/modules/auth/openid/openid.go
+++ b/pkg/modules/auth/openid/openid.go
@@ -30,6 +30,7 @@ import (
 	"code.vikunja.io/api/pkg/log"
 	"code.vikunja.io/api/pkg/models"
 	"code.vikunja.io/api/pkg/modules/auth"
+	"code.vikunja.io/api/pkg/modules/avatar"
 	"code.vikunja.io/api/pkg/modules/avatar/upload"
 	"code.vikunja.io/api/pkg/user"
 	"code.vikunja.io/api/pkg/utils"
@@ -250,6 +251,8 @@ func syncUserAvatarFromOpenID(s *xorm.Session, u *user.User, pictureURL string)
 		return fmt.Errorf("error storing avatar: %w", err)
 	}
 
+	avatar.FlushAllCaches(u)
+
 	return nil
 }
 
diff --git a/pkg/modules/avatar/avatar.go b/pkg/modules/avatar/avatar.go
index bb8bd67a8..9a7ab0f55 100644
--- a/pkg/modules/avatar/avatar.go
+++ b/pkg/modules/avatar/avatar.go
@@ -16,10 +16,40 @@
 
 package avatar
 
-import "code.vikunja.io/api/pkg/user"
+import (
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/modules/avatar/empty"
+	"code.vikunja.io/api/pkg/modules/avatar/gravatar"
+	"code.vikunja.io/api/pkg/modules/avatar/initials"
+	"code.vikunja.io/api/pkg/modules/avatar/ldap"
+	"code.vikunja.io/api/pkg/modules/avatar/marble"
+	"code.vikunja.io/api/pkg/modules/avatar/openid"
+	"code.vikunja.io/api/pkg/modules/avatar/upload"
+	"code.vikunja.io/api/pkg/user"
+)
 
 // Provider defines the avatar provider interface
 type Provider interface {
 	// GetAvatar is the method used to get an actual avatar for a user
 	GetAvatar(user *user.User, size int64) (avatar []byte, mimeType string, err error)
+	// FlushCache removes cached avatar data for the user
+	FlushCache(u *user.User) error
+}
+
+// FlushAllCaches removes cached avatars for the given user for all providers
+func FlushAllCaches(u *user.User) {
+	providers := []Provider{
+		&upload.Provider{},
+		&gravatar.Provider{},
+		&initials.Provider{},
+		&ldap.Provider{},
+		&openid.Provider{},
+		&marble.Provider{},
+		&empty.Provider{},
+	}
+	for _, p := range providers {
+		if err := p.FlushCache(u); err != nil {
+			log.Errorf("Error flushing avatar cache: %v", err)
+		}
+	}
 }
diff --git a/pkg/modules/avatar/empty/empty.go b/pkg/modules/avatar/empty/empty.go
index ebc330661..9c0134612 100644
--- a/pkg/modules/avatar/empty/empty.go
+++ b/pkg/modules/avatar/empty/empty.go
@@ -22,6 +22,9 @@ import "code.vikunja.io/api/pkg/user"
 type Provider struct {
 }
 
+// FlushCache is a no-op for the empty provider
+func (p *Provider) FlushCache(_ *user.User) error { return nil }
+
 const defaultAvatar string = `<?xml version="1.0" encoding="UTF-8"?>
 <svg width="128" height="128" version="1.1" viewBox="0 0 33.867 33.867" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xlink="http://www.w3.org/1999/xlink">
 <metadata>
diff --git a/pkg/modules/avatar/gravatar/gravatar.go b/pkg/modules/avatar/gravatar/gravatar.go
index 43c4fc50c..6af3d148f 100644
--- a/pkg/modules/avatar/gravatar/gravatar.go
+++ b/pkg/modules/avatar/gravatar/gravatar.go
@@ -41,6 +41,11 @@ type avatar struct {
 type Provider struct {
 }
 
+// FlushCache removes all gravatar cache entries for a user
+func (g *Provider) FlushCache(u *user.User) error {
+	return keyvalue.DelPrefix(keyPrefix + u.Username + "_")
+}
+
 const keyPrefix = "gravatar_avatar_"
 
 // GetAvatar implements getting the avatar for the user
diff --git a/pkg/modules/avatar/initials/initials.go b/pkg/modules/avatar/initials/initials.go
index aa81aaa02..da5aa3991 100644
--- a/pkg/modules/avatar/initials/initials.go
+++ b/pkg/modules/avatar/initials/initials.go
@@ -39,6 +39,14 @@ import (
 type Provider struct {
 }
 
+// FlushCache removes cached initials avatars for a user
+func (p *Provider) FlushCache(u *user.User) error {
+	if err := keyvalue.Del(getCacheKey("full", u.ID)); err != nil {
+		return err
+	}
+	return keyvalue.DelPrefix(getCacheKey("resized", u.ID))
+}
+
 var (
 	avatarBgColors = []*color.RGBA{
 		{R: 69, G: 189, B: 243, A: 255},
diff --git a/pkg/modules/avatar/ldap/ldap.go b/pkg/modules/avatar/ldap/ldap.go
index aba3e0442..4a4e24d37 100644
--- a/pkg/modules/avatar/ldap/ldap.go
+++ b/pkg/modules/avatar/ldap/ldap.go
@@ -28,3 +28,8 @@ func (p *Provider) GetAvatar(user *user.User, size int64) (avatar []byte, mimeTy
 
 	return up.GetAvatar(user, size)
 }
+
+func (p *Provider) FlushCache(u *user.User) error {
+	up := upload.Provider{}
+	return up.FlushCache(u)
+}
diff --git a/pkg/modules/avatar/marble/marble.go b/pkg/modules/avatar/marble/marble.go
index c7a3bd393..8a89ac106 100644
--- a/pkg/modules/avatar/marble/marble.go
+++ b/pkg/modules/avatar/marble/marble.go
@@ -27,6 +27,9 @@ import (
 type Provider struct {
 }
 
+// FlushCache is a no-op for the marble provider
+func (p *Provider) FlushCache(_ *user.User) error { return nil }
+
 const avatarSize = 80
 
 var colors = []string{
diff --git a/pkg/modules/avatar/openid/openid.go b/pkg/modules/avatar/openid/openid.go
index b950a6e45..d7398b540 100644
--- a/pkg/modules/avatar/openid/openid.go
+++ b/pkg/modules/avatar/openid/openid.go
@@ -28,3 +28,8 @@ func (p *Provider) GetAvatar(user *user.User, size int64) (avatar []byte, mimeTy
 
 	return up.GetAvatar(user, size)
 }
+
+func (p *Provider) FlushCache(u *user.User) error {
+	up := upload.Provider{}
+	return up.FlushCache(u)
+}
diff --git a/pkg/modules/avatar/upload/upload.go b/pkg/modules/avatar/upload/upload.go
index fac4c799a..71c0754d3 100644
--- a/pkg/modules/avatar/upload/upload.go
+++ b/pkg/modules/avatar/upload/upload.go
@@ -36,6 +36,12 @@ import (
 type Provider struct {
 }
 
+// FlushCache removes cached avatars for a user
+func (p *Provider) FlushCache(u *user.User) error {
+	InvalidateCache(u)
+	return nil
+}
+
 // CachedAvatar represents a cached avatar with its content and mime type
 type CachedAvatar struct {
 	Content  []byte
diff --git a/pkg/routes/api/v1/avatar.go b/pkg/routes/api/v1/avatar.go
index a08ea62ef..53e8e8b8e 100644
--- a/pkg/routes/api/v1/avatar.go
+++ b/pkg/routes/api/v1/avatar.go
@@ -177,5 +177,7 @@ func UploadAvatar(c echo.Context) (err error) {
 		return handler.HandleHTTPError(err)
 	}
 
+	avatar.FlushAllCaches(u)
+
 	return c.JSON(http.StatusOK, models.Message{Message: "Avatar was uploaded successfully."})
 }
diff --git a/pkg/routes/api/v1/user_settings.go b/pkg/routes/api/v1/user_settings.go
index 4a0d1c32e..5d31c8089 100644
--- a/pkg/routes/api/v1/user_settings.go
+++ b/pkg/routes/api/v1/user_settings.go
@@ -26,6 +26,7 @@ import (
 
 	"code.vikunja.io/api/pkg/db"
 	"code.vikunja.io/api/pkg/models"
+	"code.vikunja.io/api/pkg/modules/avatar"
 	user2 "code.vikunja.io/api/pkg/user"
 	"code.vikunja.io/api/pkg/web/handler"
 )
@@ -133,6 +134,8 @@ func ChangeUserAvatarProvider(c echo.Context) error {
 		return handler.HandleHTTPError(err)
 	}
 
+	oldProvider := user.AvatarProvider
+
 	user.AvatarProvider = uap.AvatarProvider
 
 	_, err = user2.UpdateUser(s, user, false)
@@ -146,6 +149,10 @@ func ChangeUserAvatarProvider(c echo.Context) error {
 		return handler.HandleHTTPError(err)
 	}
 
+	if oldProvider != user.AvatarProvider {
+		avatar.FlushAllCaches(user)
+	}
+
 	return c.JSON(http.StatusOK, &models.Message{Message: "Avatar was changed successfully."})
 }