Flutter - Custom widget animation state reversed in Gridview - flutter

Question

I have a gridview with a couple of custom cards that when selected will change color to indicate to the user that they have selected the card. When I scroll down and return to the previously selected card, the color changes like it was never selected.
At first, I thought it was a key issue but that didn't solve anything.
This is the code I have
// Code for the Gridview
GridView.builder(
padding: EdgeInsets.all(Insets.sm),
gridDelegate: SliverGridDelegateWithFixedCrossAxisCount(
crossAxisCount: 2,
mainAxisSpacing: Insets.sm,
crossAxisSpacing: Insets.sm,
),
itemCount: categories.length,
itemBuilder: (BuildContext context, int index) {
return CategoryCard(
key: UniqueKey(),
category: categories[index],
onTap: (isSelected) {
(isSelected)
? game.categories.add(categories[index])
: game.categories.remove(categories[index]);
},
);
},
),
// This is the code for the custom widget
class CategoryCard extends StatefulWidget {
const CategoryCard({
super.key,
required this.category,
required this.onTap,
});
final Category category;
final Function(bool) onTap;
#override
State<CategoryCard> createState() => _CategoryCardState();
}
class _CategoryCardState extends State<CategoryCard>
with TickerProviderStateMixin {
late AnimationController controller;
late Animation<Color?> containerBackground;
late Animation<Color?> imageBackground;
late Animation<Color?> textColor;
late CurvedAnimation curve;
bool isSelected = true;
#override
void initState() {
super.initState();
controller = AnimationController(
duration: const Duration(milliseconds: 300),
vsync: this,
);
curve = CurvedAnimation(
parent: controller,
curve: Curves.easeInOut,
);
containerBackground =
ColorTween(begin: Colors.white, end: AppColors.blue200)
.animate(controller)
..addListener(() => setState(() {}));
imageBackground =
ColorTween(begin: AppColors.blue50, end: AppColors.blue100)
.animate(controller)
..addListener(() => setState(() {}));
textColor = ColorTween(begin: AppColors.blue300, end: Colors.white)
.animate(controller)
..addListener(() => setState(() {}));
}
#override
void dispose() {
controller.dispose();
super.dispose();
}
void animateColor() {
if (isSelected) {
controller.forward();
} else {
controller.reverse();
}
widget.onTap(isSelected);
isSelected = !isSelected;
}
#override
Widget build(BuildContext context) {
return GestureDetector(
onTap: () => {animateColor()},
child: Container(
decoration: BoxDecoration(
color: containerBackground.value,
borderRadius: Corners.lgBorder,
),
child: Column(mainAxisAlignment: MainAxisAlignment.center, children: [
Container(
padding: EdgeInsets.all(Insets.xs),
decoration: BoxDecoration(
color: imageBackground.value,
borderRadius: Corners.lgBorder,
),
clipBehavior: Clip.antiAlias,
child: Image.asset(
"assets/images/${widget.category.name.toLowerCase()}.png",
width: 78,
height: 78,
),
),
SizedBox(height: Insets.med),
Text(
widget.category.name,
style: TextStyles.body1.copyWith(color: textColor.value),
)
]),
),
);
}
}
Any help or explanation on why this is happening is appreciated.