diff --git a/api/src/main/java/com/velocitypowered/api/scoreboard/ProxyScoreboard.java b/api/src/main/java/com/velocitypowered/api/scoreboard/ProxyScoreboard.java
index 0e64967..4ffa9d4 100644
--- a/api/src/main/java/com/velocitypowered/api/scoreboard/ProxyScoreboard.java
+++ b/api/src/main/java/com/velocitypowered/api/scoreboard/ProxyScoreboard.java
@@ -127,6 +127,17 @@ default ProxyObjective createObjective(@NonNull String name, @NonNull Consumer<P
     @NotNull
     Collection<ProxyTeam> getTeams();
 
+    /**
+     * Returns team which the given entry is a member of. If the entry is
+     * not a member of any team, returns {@code null}.
+     *
+     * @param   entry
+     *          Entry to check for team membership
+     * @return  Team which the entry is a member of, or {@code null} if the entry is not a member of any team
+     */
+    @Nullable
+    ProxyTeam getTeamByEntry(@NonNull String entry);
+
     /**
      * Creates a new team builder.
      *
diff --git a/api/src/main/java/com/velocitypowered/api/scoreboard/Scoreboard.java b/api/src/main/java/com/velocitypowered/api/scoreboard/Scoreboard.java
index e2ff02d..419bb88 100644
--- a/api/src/main/java/com/velocitypowered/api/scoreboard/Scoreboard.java
+++ b/api/src/main/java/com/velocitypowered/api/scoreboard/Scoreboard.java
@@ -75,4 +75,15 @@ public interface Scoreboard {
      */
     @NotNull
     Collection<? extends Team> getTeams();
+
+    /**
+     * Returns team which the given entry is a member of. If the entry is
+     * not a member of any team, returns {@code null}.
+     *
+     * @param   entry
+     *          Entry to check for team membership
+     * @return  Team which the entry is a member of, or {@code null} if the entry is not a member of any team
+     */
+    @Nullable
+    Team getTeamByEntry(@NonNull String entry);
 }
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/data/PacketHandler.java b/proxy/src/main/java/com/velocitypowered/proxy/data/PacketHandler.java
index f886d89..a77aa61 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/data/PacketHandler.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/data/PacketHandler.java
@@ -226,8 +226,19 @@ public static boolean handle(@NonNull MinecraftSessionHandler handler, @NonNull
         } else {
             // Remove all entries occupied by a proxy team
             if (packet.getEntries() != null) { // Any player action
-                for (VelocityTeam proxyTeam : getProxy(handler).getTeamsRaw()) {
-                    packet.getEntries().removeAll(proxyTeam.getEntryCollection());
+                VelocityScoreboard scoreboard = getProxy(handler);
+                if (packet.getEntries().getEntry() != null) {
+                    VelocityTeam teamByEntry = scoreboard.getTeamByEntry(packet.getEntries().getEntry());
+                    if (teamByEntry != null) {
+                        packet.getEntries().remove(packet.getEntries().getEntry());
+                    }
+                } else {
+                    for (String entry : packet.getEntries().getEntries()) {
+                        VelocityTeam teamByEntry = scoreboard.getTeamByEntry(entry);
+                        if (teamByEntry != null) {
+                            packet.getEntries().remove(entry);
+                        }
+                    }
                 }
             }
         }
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/VelocityScoreboard.java b/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/VelocityScoreboard.java
index 84f8b97..2081857 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/VelocityScoreboard.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/VelocityScoreboard.java
@@ -152,9 +152,10 @@ public Collection<ProxyTeam> getTeams() {
         return Collections.unmodifiableCollection(teams.values());
     }
 
-    @NotNull
-    public Collection<VelocityTeam> getTeamsRaw() {
-        return teams.values();
+    @Override
+    @Nullable
+    public VelocityTeam getTeamByEntry(@NonNull String entry) {
+        return teamEntries.get(entry);
     }
 
     @ApiStatus.Internal
diff --git a/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/downstream/DownstreamScoreboard.java b/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/downstream/DownstreamScoreboard.java
index 7b6d0e0..c70c96c 100644
--- a/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/downstream/DownstreamScoreboard.java
+++ b/proxy/src/main/java/com/velocitypowered/proxy/scoreboard/downstream/DownstreamScoreboard.java
@@ -314,17 +314,24 @@ public boolean handle(@NonNull TeamPacket packet) {
                         packet.getName(),
                         Collections.unmodifiableCollection(packet.getEntries().getEntries())
                 ));
-                for (DownstreamTeam allTeams : teams.values()) {
-                    allTeams.getEntryCollection().removeAll(entries);
-                }
-                team.addEntries(entries);
+
                 if (entries.getEntry() != null) {
-                    teamEntries.put(entries.getEntry(), team);
+                    String entry = entries.getEntry();
+                    DownstreamTeam previous = getTeamByEntry(entry);
+                    if (previous != null) {
+                        previous.getEntryCollection().remove(entry);
+                    }
+                    teamEntries.put(entry, team);
                 } else {
                     for (String entry : entries.getEntries()) {
+                        DownstreamTeam previous = getTeamByEntry(entry);
+                        if (previous != null) {
+                            previous.getEntryCollection().remove(entry);
+                        }
                         teamEntries.put(entry, team);
                     }
                 }
+                team.addEntries(entries);
             }
             case REMOVE_PLAYER -> {
                 eventSource.fireEvent(new TeamEvent.RemovePlayers(
@@ -376,14 +383,7 @@ public Collection<? extends Team> getTeams() {
         return Collections.unmodifiableCollection(teams.values());
     }
 
-    /**
-     * Returns team the specified entry belongs to. If it does not belong to any,
-     * {@code null} is returned.
-     *
-     * @param   entry
-     *          Entry to get team of
-     * @return  Team where this entry belongs or {@code null} if none
-     */
+    @Override
     @Nullable
     public DownstreamTeam getTeamByEntry(@NonNull String entry) {
         return teamEntries.get(entry);