How to prevent camera from going through specific layers - unity3d

Question

I have a camera controller that can move, zoom and rotate the camera. It's pretty much the exact camera controller from Cities Skyline. i.e top-down with some freedom of movement.
I am using CheckSphere to detect when the camera collides with wall/ground layers, and this works perfectly. However my issue is I do not know how to reset it's position to the last valid one, and to keep it there until the camera chooses a valid direction. At the moment as soon as my CheckSphere hits a wall, the camera is stuck and no longer responds to input.
I tried to store the last valid position, rotation and zoom and simply force this when CheckSphere hits something, but it's not working.
Is there some way I can can use this method with CheckSphere in a way that the camera won't get stuck every time it hits something it's not allowed to hit?
void HandleInput()
{
if (Physics.CheckSphere(cameraTransform.position, 7, collideWith))
{
newPosition = oldPosition;
newRotation = oldRotation;
newZoom = oldZoom;
Debug.Log("Did Hit");
}
else
{
//Speed controls
if (Input.GetKey(KeyCode.LeftShift))
{
movementSpeed = fastSpeed;
}
else
{
movementSpeed = normalSpeed;
}
// Adjust movement speed based on camera zoom
movementSpeed *= (cameraTransform.localPosition.y / zoomSpeedFactor);
Vector3 adjustedForward = transform.forward;
adjustedForward.y = 0;
//Movement controls
if (Input.GetKey(KeyCode.W) || Input.GetKey(KeyCode.UpArrow))
{
newPosition += (adjustedForward * movementSpeed);
}
if (Input.GetKey(KeyCode.S) || Input.GetKey(KeyCode.DownArrow))
{
newPosition += (adjustedForward * -movementSpeed);
}
if (Input.GetKey(KeyCode.D) || Input.GetKey(KeyCode.RightArrow))
{
newPosition += (transform.right * movementSpeed);
}
if (Input.GetKey(KeyCode.A) || Input.GetKey(KeyCode.LeftArrow))
{
newPosition += (transform.right * -movementSpeed);
}
//Rotation controls
if (Input.GetMouseButtonDown(2))
{
rotateStartPosition = Input.mousePosition;
}
if (Input.GetMouseButton(2))
{
rotateCurrentPosition = Input.mousePosition;
Vector3 difference = rotateStartPosition - rotateCurrentPosition;
rotateStartPosition = rotateCurrentPosition;
newRotation *= Quaternion.Euler(Vector3.up * (-difference.x / 5f));
newRotation *= Quaternion.Euler(Vector3.right * (difference.y / 5f));
Vector3 euler = newRotation.eulerAngles;
euler.x = ClampAngle(euler.x, verticalRotationClamp.x, verticalRotationClamp.y);
euler.z = 0;
newRotation.eulerAngles = euler;
}
//Zoom controls
if (Input.mouseScrollDelta.y != 0 && !EventSystem.current.IsPointerOverGameObject())
{
newZoom -= Input.mouseScrollDelta.y * zoomAmount;
newZoom.y = ClampValue(newZoom.y, zoomClamp.x, zoomClamp.y);
newZoom.z = ClampValue(newZoom.z, -zoomClamp.y, -zoomClamp.x);
}
oldPosition = new Vector3(newPosition.x, newPosition.y, newPosition.z);
oldRotation = new Quaternion(newRotation.x, newRotation.y, newRotation.z, newRotation.w);
oldZoom = new Vector3(newZoom.x, newZoom.y, newZoom.z);
Debug.Log("Did not Hit");
}
transform.position = Vector3.Lerp(transform.position, newPosition, Time.unscaledDeltaTime * acceleration);
transform.rotation = Quaternion.Lerp(transform.rotation, newRotation, Time.unscaledDeltaTime * acceleration);
cameraTransform.localPosition = Vector3.Lerp(cameraTransform.localPosition, newZoom, Time.unscaledDeltaTime * acceleration);
}