diff --git a/go.mod b/go.mod
index 5fb2a354..e8973f3f 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,6 @@ require (
 	golang.org/x/sync v0.6.0 // indirect
 )
 
-replace github.com/notaryproject/notation-core-go => github.com/Two-Hearts/notation-core-go v0.0.0-20240322064059-bf4ea64638b0
+replace github.com/notaryproject/notation-core-go => github.com/Two-Hearts/notation-core-go v0.0.0-20240322074029-e6537801a769
 
 replace github.com/notaryproject/tspclient-go => github.com/Two-Hearts/tspclient-go v0.0.0-20240322031047-c33159600668
diff --git a/go.sum b/go.sum
index 0dfc6f09..5b61fa77 100644
--- a/go.sum
+++ b/go.sum
@@ -1,7 +1,7 @@
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/Two-Hearts/notation-core-go v0.0.0-20240322064059-bf4ea64638b0 h1:CRM7IOv+86vhe82iD8EAGRbi/V+BhTXIEFwGegCP9uE=
-github.com/Two-Hearts/notation-core-go v0.0.0-20240322064059-bf4ea64638b0/go.mod h1:cYwg3vrJsiuSC3ID7bG4/q6spGYbBTIr2mqG3ePwrqQ=
+github.com/Two-Hearts/notation-core-go v0.0.0-20240322074029-e6537801a769 h1:IjW5HyuNFL1rW29o/dCFoO4J5kXGCrEMOwNTwPyd6fs=
+github.com/Two-Hearts/notation-core-go v0.0.0-20240322074029-e6537801a769/go.mod h1:cYwg3vrJsiuSC3ID7bG4/q6spGYbBTIr2mqG3ePwrqQ=
 github.com/Two-Hearts/tspclient-go v0.0.0-20240322031047-c33159600668 h1:DwEjNM07LP9yYT17LMWEgv4g0UnjmORuyX2aqUgnURE=
 github.com/Two-Hearts/tspclient-go v0.0.0-20240322031047-c33159600668/go.mod h1:Pgt9nPf69t08eVXdxjcfxZalElbQocRuP1DGSKZDpMs=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
diff --git a/verifier/verifier.go b/verifier/verifier.go
index d2a1a5de..642ebefc 100644
--- a/verifier/verifier.go
+++ b/verifier/verifier.go
@@ -516,111 +516,118 @@ func verifyExpiry(outcome *notation.VerificationOutcome) *notation.ValidationRes
 }
 
 func verifyAuthenticTimestamp(ctx context.Context, trustPolicy *trustpolicy.TrustPolicy, x509TrustStore truststore.X509TrustStore, outcome *notation.VerificationOutcome) *notation.ValidationResult {
-	invalidTimestamp := false
-	var err error
-
 	if signerInfo := outcome.EnvelopeContent.SignerInfo; signerInfo.SignedAttributes.SigningScheme == signature.SigningSchemeX509 {
-		var timeStampLowerLimit time.Time
-		var timeStampUpperLimit time.Time
-		// TODO verify RFC3161 TSA signature if present (not in RC1)
-		// https://github.com/notaryproject/notation-go/issues/78
+		var needTimestamp bool
+		for _, cert := range signerInfo.CertificateChain {
+			if time.Now().Before(cert.NotBefore) || time.Now().After(cert.NotAfter) {
+				// found at least one cert that current time is not in its
+				// validity period; need timestamp to continue this step
+				needTimestamp = true
+				break
+			}
+		}
+		if !needTimestamp { // this step is a success
+			return &notation.ValidationResult{
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
+			}
+		}
 		if len(signerInfo.UnsignedAttributes.TimestampSignature) == 0 {
-			// if there is no TSA signature, then every certificate should be
-			// valid at the time of verification
-			timeStampLowerLimit = time.Now()
-			timeStampUpperLimit = timeStampLowerLimit
-		} else {
-			trustTSACerts, err := loadX509TSATrustStores(ctx, outcome.EnvelopeContent.SignerInfo.SignedAttributes.SigningScheme, trustPolicy, x509TrustStore)
-			if err != nil {
-				return &notation.ValidationResult{
-					Error:  err,
-					Type:   trustpolicy.TypeAuthenticTimestamp,
-					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
-				}
+			// if there is no timestamp token, fail this step
+			return &notation.ValidationResult{
+				Error:  errors.New("current time is not in certificate chain validity period and no timestamp token was found in the signature envelope"),
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			if len(trustTSACerts) < 1 {
-				return &notation.ValidationResult{
-					Error:  notation.ErrorVerificationInconclusive{Msg: "no trusted TSA certificate was found to verify authentic timestamp"},
-					Type:   trustpolicy.TypeAuthenticTimestamp,
-					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
-				}
+		}
+		trustTSACerts, err := loadX509TSATrustStores(ctx, outcome.EnvelopeContent.SignerInfo.SignedAttributes.SigningScheme, trustPolicy, x509TrustStore)
+		if err != nil {
+			return &notation.ValidationResult{
+				Error:  err,
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			signedToken, err := tspclient.ParseSignedToken(ctx, signerInfo.UnsignedAttributes.TimestampSignature)
-			if err != nil {
-				return &notation.ValidationResult{
-					Error:  err,
-					Type:   trustpolicy.TypeAuthenticTimestamp,
-					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
-				}
+		}
+		if len(trustTSACerts) < 1 {
+			return &notation.ValidationResult{
+				Error:  errors.New("no trusted TSA root certificate was found in the trust store"),
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			roots := x509.NewCertPool()
-			for _, cert := range trustTSACerts {
-				roots.AddCert(cert)
+		}
+		signedToken, err := tspclient.ParseSignedToken(ctx, signerInfo.UnsignedAttributes.TimestampSignature)
+		if err != nil {
+			return &notation.ValidationResult{
+				Error:  err,
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
+			}
+		}
+		roots := x509.NewCertPool()
+		for _, cert := range trustTSACerts {
+			roots.AddCert(cert)
+		}
+		opts := x509.VerifyOptions{
+			Roots: roots,
+		}
+		// TODO: check revocation of cert chain
+		if _, err := signedToken.Verify(ctx, opts); err != nil {
+			return &notation.ValidationResult{
+				Error:  err,
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			opts := x509.VerifyOptions{
-				Roots: roots,
+		}
+		info, err := signedToken.Info()
+		if err != nil {
+			return &notation.ValidationResult{
+				Error:  err,
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			if _, err := signedToken.Verify(ctx, opts); err != nil {
-				return &notation.ValidationResult{
-					Error:  err,
-					Type:   trustpolicy.TypeAuthenticTimestamp,
-					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
-				}
+		}
+		if err := info.VerifyContent(signerInfo.Signature); err != nil {
+			return &notation.ValidationResult{
+				Error:  err,
+				Type:   trustpolicy.TypeAuthenticTimestamp,
+				Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 			}
-			info, err := signedToken.Info()
-			if err != nil {
+		}
+		// consume the timestamp
+		ts, accuracy := info.Timestamp()
+		timeStampLowerLimit := ts.Add(-accuracy)
+		timeStampUpperLimit := ts.Add(accuracy)
+		fmt.Printf("timestamp token time range: [%v, %v]\n", timeStampLowerLimit, timeStampUpperLimit)
+		for _, cert := range signerInfo.CertificateChain {
+			if timeStampLowerLimit.Before(cert.NotBefore) {
 				return &notation.ValidationResult{
-					Error:  err,
+					Error:  fmt.Errorf("timestamp lower limit %q is before certificate %q validity period , it will be valid from %q", timeStampLowerLimit.Format(time.RFC1123Z), cert.Subject, cert.NotBefore.Format(time.RFC1123Z)),
 					Type:   trustpolicy.TypeAuthenticTimestamp,
 					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 				}
 			}
-			if err := info.VerifyContent(signerInfo.Signature); err != nil {
+			if timeStampUpperLimit.After(cert.NotAfter) {
 				return &notation.ValidationResult{
-					Error:  err,
+					Error:  fmt.Errorf("timestamp upper limit %q is after certificate %q validity period, it was expired at %q", timeStampUpperLimit.Format(time.RFC1123Z), cert.Subject, cert.NotAfter.Format(time.RFC1123Z)),
 					Type:   trustpolicy.TypeAuthenticTimestamp,
 					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
 				}
 			}
-			ts, accuracy := info.Timestamp()
-			timeStampLowerLimit = ts.Add(-accuracy)
-			timeStampUpperLimit = ts.Add(accuracy)
-			fmt.Printf("timestamp token time range: [%v, %v]\n", timeStampLowerLimit, timeStampUpperLimit)
-		}
-		for _, cert := range signerInfo.CertificateChain {
-			fmt.Printf("cert validiy time range: [%v, %v]\n", cert.NotBefore, cert.NotAfter)
-			if timeStampLowerLimit.Before(cert.NotBefore) {
-				invalidTimestamp = true
-				err = fmt.Errorf("certificate %q is not valid yet, it will be valid from %q", cert.Subject, cert.NotBefore.Format(time.RFC1123Z))
-				break
-			}
-			if timeStampUpperLimit.After(cert.NotAfter) {
-				invalidTimestamp = true
-				err = fmt.Errorf("certificate %q is not valid anymore, it was expired at %q", cert.Subject, cert.NotAfter.Format(time.RFC1123Z))
-				break
-			}
 		}
 	} else if signerInfo.SignedAttributes.SigningScheme == signature.SigningSchemeX509SigningAuthority {
 		authenticSigningTime := signerInfo.SignedAttributes.SigningTime
-		// TODO use authenticSigningTime from signerInfo
-		// https://github.com/notaryproject/notation-core-go/issues/38
 		for _, cert := range signerInfo.CertificateChain {
 			if authenticSigningTime.Before(cert.NotBefore) || authenticSigningTime.After(cert.NotAfter) {
-				invalidTimestamp = true
-				err = fmt.Errorf("certificate %q was not valid when the digital signature was produced at %q", cert.Subject, authenticSigningTime.Format(time.RFC1123Z))
-				break
+				return &notation.ValidationResult{
+					Error:  fmt.Errorf("certificate %q was not valid when the digital signature was produced at %q", cert.Subject, authenticSigningTime.Format(time.RFC1123Z)),
+					Type:   trustpolicy.TypeAuthenticTimestamp,
+					Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
+				}
 			}
 		}
 	}
 
-	if invalidTimestamp {
-		return &notation.ValidationResult{
-			Error:  err,
-			Type:   trustpolicy.TypeAuthenticTimestamp,
-			Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],
-		}
-	}
-
+	// this step is a success
 	return &notation.ValidationResult{
 		Type:   trustpolicy.TypeAuthenticTimestamp,
 		Action: outcome.VerificationLevel.Enforcement[trustpolicy.TypeAuthenticTimestamp],